1 /*
    2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "gc/shared/collectedHeap.inline.hpp"
   30 #include "interpreter/interpreter.hpp"
   31 #include "memory/resourceArea.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/macros.hpp"
   39 
   40 #ifdef PRODUCT
   41 #define BLOCK_COMMENT(str) /* nothing */
   42 #define STOP(error) stop(error)
   43 #else
   44 #define BLOCK_COMMENT(str) block_comment(str)
   45 #define STOP(error) block_comment(error); stop(error)
   46 #endif
   47 
   48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   49 // Implementation of AddressLiteral
   50 
   51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   53   // -----------------Table 4.5 -------------------- //
   54   16, 32, 64,  // EVEX_FV(0)
   55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   58   8,  16, 32,  // EVEX_HV(0)
   59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   60   // -----------------Table 4.6 -------------------- //
   61   16, 32, 64,  // EVEX_FVM(0)
   62   1,  1,  1,   // EVEX_T1S(0)
   63   2,  2,  2,   // EVEX_T1S(1)
   64   4,  4,  4,   // EVEX_T1S(2)
   65   8,  8,  8,   // EVEX_T1S(3)
   66   4,  4,  4,   // EVEX_T1F(0)
   67   8,  8,  8,   // EVEX_T1F(1)
   68   8,  8,  8,   // EVEX_T2(0)
   69   0,  16, 16,  // EVEX_T2(1)
   70   0,  16, 16,  // EVEX_T4(0)
   71   0,  0,  32,  // EVEX_T4(1)
   72   0,  0,  32,  // EVEX_T8(0)
   73   8,  16, 32,  // EVEX_HVM(0)
   74   4,  8,  16,  // EVEX_QVM(0)
   75   2,  4,  8,   // EVEX_OVM(0)
   76   16, 16, 16,  // EVEX_M128(0)
   77   8,  32, 64,  // EVEX_DUP(0)
   78   0,  0,  0    // EVEX_NTUP
   79 };
   80 
   81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   82   _is_lval = false;
   83   _target = target;
   84   switch (rtype) {
   85   case relocInfo::oop_type:
   86   case relocInfo::metadata_type:
   87     // Oops are a special case. Normally they would be their own section
   88     // but in cases like icBuffer they are literals in the code stream that
   89     // we don't have a section for. We use none so that we get a literal address
   90     // which is always patchable.
   91     break;
   92   case relocInfo::external_word_type:
   93     _rspec = external_word_Relocation::spec(target);
   94     break;
   95   case relocInfo::internal_word_type:
   96     _rspec = internal_word_Relocation::spec(target);
   97     break;
   98   case relocInfo::opt_virtual_call_type:
   99     _rspec = opt_virtual_call_Relocation::spec();
  100     break;
  101   case relocInfo::static_call_type:
  102     _rspec = static_call_Relocation::spec();
  103     break;
  104   case relocInfo::runtime_call_type:
  105     _rspec = runtime_call_Relocation::spec();
  106     break;
  107   case relocInfo::poll_type:
  108   case relocInfo::poll_return_type:
  109     _rspec = Relocation::spec_simple(rtype);
  110     break;
  111   case relocInfo::none:
  112     break;
  113   default:
  114     ShouldNotReachHere();
  115     break;
  116   }
  117 }
  118 
  119 // Implementation of Address
  120 
  121 #ifdef _LP64
  122 
  123 Address Address::make_array(ArrayAddress adr) {
  124   // Not implementable on 64bit machines
  125   // Should have been handled higher up the call chain.
  126   ShouldNotReachHere();
  127   return Address();
  128 }
  129 
  130 // exceedingly dangerous constructor
  131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  132   _base  = noreg;
  133   _index = noreg;
  134   _scale = no_scale;
  135   _disp  = disp;
  136   _xmmindex = xnoreg;
  137   _isxmmindex = false;
  138   switch (rtype) {
  139     case relocInfo::external_word_type:
  140       _rspec = external_word_Relocation::spec(loc);
  141       break;
  142     case relocInfo::internal_word_type:
  143       _rspec = internal_word_Relocation::spec(loc);
  144       break;
  145     case relocInfo::runtime_call_type:
  146       // HMM
  147       _rspec = runtime_call_Relocation::spec();
  148       break;
  149     case relocInfo::poll_type:
  150     case relocInfo::poll_return_type:
  151       _rspec = Relocation::spec_simple(rtype);
  152       break;
  153     case relocInfo::none:
  154       break;
  155     default:
  156       ShouldNotReachHere();
  157   }
  158 }
  159 #else // LP64
  160 
  161 Address Address::make_array(ArrayAddress adr) {
  162   AddressLiteral base = adr.base();
  163   Address index = adr.index();
  164   assert(index._disp == 0, "must not have disp"); // maybe it can?
  165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  166   array._rspec = base._rspec;
  167   return array;
  168 }
  169 
  170 // exceedingly dangerous constructor
  171 Address::Address(address loc, RelocationHolder spec) {
  172   _base  = noreg;
  173   _index = noreg;
  174   _scale = no_scale;
  175   _disp  = (intptr_t) loc;
  176   _rspec = spec;
  177   _xmmindex = xnoreg;
  178   _isxmmindex = false;
  179 }
  180 
  181 #endif // _LP64
  182 
  183 
  184 
  185 // Convert the raw encoding form into the form expected by the constructor for
  186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  187 // that to noreg for the Address constructor.
  188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  189   RelocationHolder rspec;
  190   if (disp_reloc != relocInfo::none) {
  191     rspec = Relocation::spec_simple(disp_reloc);
  192   }
  193   bool valid_index = index != rsp->encoding();
  194   if (valid_index) {
  195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  196     madr._rspec = rspec;
  197     return madr;
  198   } else {
  199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  200     madr._rspec = rspec;
  201     return madr;
  202   }
  203 }
  204 
  205 // Implementation of Assembler
  206 
  207 int AbstractAssembler::code_fill_byte() {
  208   return (u_char)'\xF4'; // hlt
  209 }
  210 
  211 void Assembler::init_attributes(void) {
  212   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  213   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  214   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  215   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  216   NOT_LP64(_is_managed = false;)
  217   _attributes = NULL;
  218 }
  219 
  220 
  221 void Assembler::membar(Membar_mask_bits order_constraint) {
  222   // We only have to handle StoreLoad
  223   if (order_constraint & StoreLoad) {
  224     // All usable chips support "locked" instructions which suffice
  225     // as barriers, and are much faster than the alternative of
  226     // using cpuid instruction. We use here a locked add [esp-C],0.
  227     // This is conveniently otherwise a no-op except for blowing
  228     // flags, and introducing a false dependency on target memory
  229     // location. We can't do anything with flags, but we can avoid
  230     // memory dependencies in the current method by locked-adding
  231     // somewhere else on the stack. Doing [esp+C] will collide with
  232     // something on stack in current method, hence we go for [esp-C].
  233     // It is convenient since it is almost always in data cache, for
  234     // any small C.  We need to step back from SP to avoid data
  235     // dependencies with other things on below SP (callee-saves, for
  236     // example). Without a clear way to figure out the minimal safe
  237     // distance from SP, it makes sense to step back the complete
  238     // cache line, as this will also avoid possible second-order effects
  239     // with locked ops against the cache line. Our choice of offset
  240     // is bounded by x86 operand encoding, which should stay within
  241     // [-128; +127] to have the 8-byte displacement encoding.
  242     //
  243     // Any change to this code may need to revisit other places in
  244     // the code where this idiom is used, in particular the
  245     // orderAccess code.
  246 
  247     int offset = -VM_Version::L1_line_size();
  248     if (offset < -128) {
  249       offset = -128;
  250     }
  251 
  252     lock();
  253     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  254   }
  255 }
  256 
  257 // make this go away someday
  258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  259   if (rtype == relocInfo::none)
  260     emit_int32(data);
  261   else
  262     emit_data(data, Relocation::spec_simple(rtype), format);
  263 }
  264 
  265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  266   assert(imm_operand == 0, "default format must be immediate in this file");
  267   assert(inst_mark() != NULL, "must be inside InstructionMark");
  268   if (rspec.type() !=  relocInfo::none) {
  269     #ifdef ASSERT
  270       check_relocation(rspec, format);
  271     #endif
  272     // Do not use AbstractAssembler::relocate, which is not intended for
  273     // embedded words.  Instead, relocate to the enclosing instruction.
  274 
  275     // hack. call32 is too wide for mask so use disp32
  276     if (format == call32_operand)
  277       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  278     else
  279       code_section()->relocate(inst_mark(), rspec, format);
  280   }
  281   emit_int32(data);
  282 }
  283 
  284 static int encode(Register r) {
  285   int enc = r->encoding();
  286   if (enc >= 8) {
  287     enc -= 8;
  288   }
  289   return enc;
  290 }
  291 
  292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  293   assert(dst->has_byte_register(), "must have byte register");
  294   assert(isByte(op1) && isByte(op2), "wrong opcode");
  295   assert(isByte(imm8), "not a byte");
  296   assert((op1 & 0x01) == 0, "should be 8bit operation");
  297   emit_int24(op1, (op2 | encode(dst)), imm8);
  298 }
  299 
  300 
  301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  302   assert(isByte(op1) && isByte(op2), "wrong opcode");
  303   assert((op1 & 0x01) == 1, "should be 32bit operation");
  304   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  305   if (is8bit(imm32)) {
  306     emit_int24(op1 | 0x02,        // set sign bit
  307                op2 | encode(dst),
  308                imm32 & 0xFF);
  309   } else {
  310     emit_int16(op1, (op2 | encode(dst)));
  311     emit_int32(imm32);
  312   }
  313 }
  314 
  315 // Force generation of a 4 byte immediate value even if it fits into 8bit
  316 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  317   assert(isByte(op1) && isByte(op2), "wrong opcode");
  318   assert((op1 & 0x01) == 1, "should be 32bit operation");
  319   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  320   emit_int16(op1, (op2 | encode(dst)));
  321   emit_int32(imm32);
  322 }
  323 
  324 // immediate-to-memory forms
  325 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  326   assert((op1 & 0x01) == 1, "should be 32bit operation");
  327   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  328   if (is8bit(imm32)) {
  329     emit_int8(op1 | 0x02); // set sign bit
  330     emit_operand(rm, adr, 1);
  331     emit_int8(imm32 & 0xFF);
  332   } else {
  333     emit_int8(op1);
  334     emit_operand(rm, adr, 4);
  335     emit_int32(imm32);
  336   }
  337 }
  338 
  339 
  340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  341   assert(isByte(op1) && isByte(op2), "wrong opcode");
  342   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  343 }
  344 
  345 
  346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  347                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  348   int mod_idx = 0;
  349   // We will test if the displacement fits the compressed format and if so
  350   // apply the compression to the displacment iff the result is8bit.
  351   if (VM_Version::supports_evex() && is_evex_inst) {
  352     switch (cur_tuple_type) {
  353     case EVEX_FV:
  354       if ((cur_encoding & VEX_W) == VEX_W) {
  355         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  356       } else {
  357         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  358       }
  359       break;
  360 
  361     case EVEX_HV:
  362       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  363       break;
  364 
  365     case EVEX_FVM:
  366       break;
  367 
  368     case EVEX_T1S:
  369       switch (in_size_in_bits) {
  370       case EVEX_8bit:
  371         break;
  372 
  373       case EVEX_16bit:
  374         mod_idx = 1;
  375         break;
  376 
  377       case EVEX_32bit:
  378         mod_idx = 2;
  379         break;
  380 
  381       case EVEX_64bit:
  382         mod_idx = 3;
  383         break;
  384       }
  385       break;
  386 
  387     case EVEX_T1F:
  388     case EVEX_T2:
  389     case EVEX_T4:
  390       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  391       break;
  392 
  393     case EVEX_T8:
  394       break;
  395 
  396     case EVEX_HVM:
  397       break;
  398 
  399     case EVEX_QVM:
  400       break;
  401 
  402     case EVEX_OVM:
  403       break;
  404 
  405     case EVEX_M128:
  406       break;
  407 
  408     case EVEX_DUP:
  409       break;
  410 
  411     default:
  412       assert(0, "no valid evex tuple_table entry");
  413       break;
  414     }
  415 
  416     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  417       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  418       if ((disp % disp_factor) == 0) {
  419         int new_disp = disp / disp_factor;
  420         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  421           disp = new_disp;
  422         }
  423       } else {
  424         return false;
  425       }
  426     }
  427   }
  428   return (-0x80 <= disp && disp < 0x80);
  429 }
  430 
  431 
  432 bool Assembler::emit_compressed_disp_byte(int &disp) {
  433   int mod_idx = 0;
  434   // We will test if the displacement fits the compressed format and if so
  435   // apply the compression to the displacment iff the result is8bit.
  436   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  437     int evex_encoding = _attributes->get_evex_encoding();
  438     int tuple_type = _attributes->get_tuple_type();
  439     switch (tuple_type) {
  440     case EVEX_FV:
  441       if ((evex_encoding & VEX_W) == VEX_W) {
  442         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  443       } else {
  444         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  445       }
  446       break;
  447 
  448     case EVEX_HV:
  449       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  450       break;
  451 
  452     case EVEX_FVM:
  453       break;
  454 
  455     case EVEX_T1S:
  456       switch (_attributes->get_input_size()) {
  457       case EVEX_8bit:
  458         break;
  459 
  460       case EVEX_16bit:
  461         mod_idx = 1;
  462         break;
  463 
  464       case EVEX_32bit:
  465         mod_idx = 2;
  466         break;
  467 
  468       case EVEX_64bit:
  469         mod_idx = 3;
  470         break;
  471       }
  472       break;
  473 
  474     case EVEX_T1F:
  475     case EVEX_T2:
  476     case EVEX_T4:
  477       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  478       break;
  479 
  480     case EVEX_T8:
  481       break;
  482 
  483     case EVEX_HVM:
  484       break;
  485 
  486     case EVEX_QVM:
  487       break;
  488 
  489     case EVEX_OVM:
  490       break;
  491 
  492     case EVEX_M128:
  493       break;
  494 
  495     case EVEX_DUP:
  496       break;
  497 
  498     default:
  499       assert(0, "no valid evex tuple_table entry");
  500       break;
  501     }
  502 
  503     int vector_len = _attributes->get_vector_len();
  504     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  505       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  506       if ((disp % disp_factor) == 0) {
  507         int new_disp = disp / disp_factor;
  508         if (is8bit(new_disp)) {
  509           disp = new_disp;
  510         }
  511       } else {
  512         return false;
  513       }
  514     }
  515   }
  516   return is8bit(disp);
  517 }
  518 
  519 static bool is_valid_encoding(int reg_enc) {
  520   return reg_enc >= 0;
  521 }
  522 
  523 static int raw_encode(Register reg) {
  524   assert(reg == noreg || reg->is_valid(), "sanity");
  525   int reg_enc = (intptr_t)reg;
  526   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  527   return reg_enc;
  528 }
  529 
  530 static int raw_encode(XMMRegister xmmreg) {
  531   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  532   int xmmreg_enc = (intptr_t)xmmreg;
  533   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  534   return xmmreg_enc;
  535 }
  536 
  537 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  538   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  539 }
  540 
  541 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  542   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  543 }
  544 
  545 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  546   assert((mod & 3) != 0b11, "forbidden");
  547   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  548   emit_int8(modrm);
  549 }
  550 
  551 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  552                                         int disp) {
  553   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  554   emit_int16(modrm, disp & 0xFF);
  555 }
  556 
  557 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  558                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  559   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  560   int sib = sib_encoding(scale, index_enc, base_enc);
  561   emit_int16(modrm, sib);
  562 }
  563 
  564 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  565                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  566                                             int disp) {
  567   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  568   int sib = sib_encoding(scale, index_enc, base_enc);
  569   emit_int24(modrm, sib, disp & 0xFF);
  570 }
  571 
  572 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  573                                     Address::ScaleFactor scale, int disp,
  574                                     RelocationHolder const& rspec,
  575                                     int rip_relative_correction) {
  576   bool no_relocation = (rspec.type() == relocInfo::none);
  577 
  578   if (is_valid_encoding(base_enc)) {
  579     if (is_valid_encoding(index_enc)) {
  580       assert(scale != Address::no_scale, "inconsistent address");
  581       // [base + index*scale + disp]
  582       if (disp == 0 && no_relocation &&
  583           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  584         // [base + index*scale]
  585         // [00 reg 100][ss index base]
  586         emit_modrm_sib(0b00, reg_enc, 0b100,
  587                        scale, index_enc, base_enc);
  588       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  589         // [base + index*scale + imm8]
  590         // [01 reg 100][ss index base] imm8
  591         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  592                              scale, index_enc, base_enc,
  593                              disp);
  594       } else {
  595         // [base + index*scale + disp32]
  596         // [10 reg 100][ss index base] disp32
  597         emit_modrm_sib(0b10, reg_enc, 0b100,
  598                        scale, index_enc, base_enc);
  599         emit_data(disp, rspec, disp32_operand);
  600       }
  601     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  602       // [rsp + disp]
  603       if (disp == 0 && no_relocation) {
  604         // [rsp]
  605         // [00 reg 100][00 100 100]
  606         emit_modrm_sib(0b00, reg_enc, 0b100,
  607                        Address::times_1, 0b100, 0b100);
  608       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  609         // [rsp + imm8]
  610         // [01 reg 100][00 100 100] disp8
  611         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  612                              Address::times_1, 0b100, 0b100,
  613                              disp);
  614       } else {
  615         // [rsp + imm32]
  616         // [10 reg 100][00 100 100] disp32
  617         emit_modrm_sib(0b10, reg_enc, 0b100,
  618                        Address::times_1, 0b100, 0b100);
  619         emit_data(disp, rspec, disp32_operand);
  620       }
  621     } else {
  622       // [base + disp]
  623       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  624       if (disp == 0 && no_relocation &&
  625           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  626         // [base]
  627         // [00 reg base]
  628         emit_modrm(0, reg_enc, base_enc);
  629       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  630         // [base + disp8]
  631         // [01 reg base] disp8
  632         emit_modrm_disp8(0b01, reg_enc, base_enc,
  633                          disp);
  634       } else {
  635         // [base + disp32]
  636         // [10 reg base] disp32
  637         emit_modrm(0b10, reg_enc, base_enc);
  638         emit_data(disp, rspec, disp32_operand);
  639       }
  640     }
  641   } else {
  642     if (is_valid_encoding(index_enc)) {
  643       assert(scale != Address::no_scale, "inconsistent address");
  644       // base == noreg
  645       // [index*scale + disp]
  646       // [00 reg 100][ss index 101] disp32
  647       emit_modrm_sib(0b00, reg_enc, 0b100,
  648                      scale, index_enc, 0b101 /* no base */);
  649       emit_data(disp, rspec, disp32_operand);
  650     } else if (!no_relocation) {
  651       // base == noreg, index == noreg
  652       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  653       // [00 reg 101] disp32
  654 
  655       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  656       // Note that the RIP-rel. correction applies to the generated
  657       // disp field, but _not_ to the target address in the rspec.
  658 
  659       // disp was created by converting the target address minus the pc
  660       // at the start of the instruction. That needs more correction here.
  661       // intptr_t disp = target - next_ip;
  662       assert(inst_mark() != NULL, "must be inside InstructionMark");
  663       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
  664       int64_t adjusted = disp;
  665       // Do rip-rel adjustment for 64bit
  666       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  667       assert(is_simm32(adjusted),
  668              "must be 32bit offset (RIP relative address)");
  669       emit_data((int32_t) adjusted, rspec, disp32_operand);
  670 
  671     } else {
  672       // base == noreg, index == noreg, no_relocation == true
  673       // 32bit never did this, did everything as the rip-rel/disp code above
  674       // [disp] ABSOLUTE
  675       // [00 reg 100][00 100 101] disp32
  676       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  677                      Address::times_1, 0b100, 0b101);
  678       emit_data(disp, rspec, disp32_operand);
  679     }
  680   }
  681 }
  682 
  683 void Assembler::emit_operand(Register reg, Register base, Register index,
  684                              Address::ScaleFactor scale, int disp,
  685                              RelocationHolder const& rspec,
  686                              int rip_relative_correction) {
  687   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  688   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  689                       scale, disp, rspec, rip_relative_correction);
  690 
  691 }
  692 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  693                              Address::ScaleFactor scale, int disp,
  694                              RelocationHolder const& rspec) {
  695   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  696   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  697   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  698                       scale, disp, rspec);
  699 }
  700 
  701 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  702                              Address::ScaleFactor scale, int disp,
  703                              RelocationHolder const& rspec) {
  704   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  705   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  706   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  707                       scale, disp, rspec, /* rip_relative_correction */ 0);
  708 }
  709 
  710 // Secret local extension to Assembler::WhichOperand:
  711 #define end_pc_operand (_WhichOperand_limit)
  712 
  713 address Assembler::locate_operand(address inst, WhichOperand which) {
  714   // Decode the given instruction, and return the address of
  715   // an embedded 32-bit operand word.
  716 
  717   // If "which" is disp32_operand, selects the displacement portion
  718   // of an effective address specifier.
  719   // If "which" is imm64_operand, selects the trailing immediate constant.
  720   // If "which" is call32_operand, selects the displacement of a call or jump.
  721   // Caller is responsible for ensuring that there is such an operand,
  722   // and that it is 32/64 bits wide.
  723 
  724   // If "which" is end_pc_operand, find the end of the instruction.
  725 
  726   address ip = inst;
  727   bool is_64bit = false;
  728 
  729   debug_only(bool has_disp32 = false);
  730   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  731 
  732   again_after_prefix:
  733   switch (0xFF & *ip++) {
  734 
  735   // These convenience macros generate groups of "case" labels for the switch.
  736 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  737 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  738              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  739 #define REP16(x) REP8((x)+0): \
  740               case REP8((x)+8)
  741 
  742   case CS_segment:
  743   case SS_segment:
  744   case DS_segment:
  745   case ES_segment:
  746   case FS_segment:
  747   case GS_segment:
  748     // Seems dubious
  749     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  750     assert(ip == inst+1, "only one prefix allowed");
  751     goto again_after_prefix;
  752 
  753   case 0x67:
  754   case REX:
  755   case REX_B:
  756   case REX_X:
  757   case REX_XB:
  758   case REX_R:
  759   case REX_RB:
  760   case REX_RX:
  761   case REX_RXB:
  762     NOT_LP64(assert(false, "64bit prefixes"));
  763     goto again_after_prefix;
  764 
  765   case REX_W:
  766   case REX_WB:
  767   case REX_WX:
  768   case REX_WXB:
  769   case REX_WR:
  770   case REX_WRB:
  771   case REX_WRX:
  772   case REX_WRXB:
  773     NOT_LP64(assert(false, "64bit prefixes"));
  774     is_64bit = true;
  775     goto again_after_prefix;
  776 
  777   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  778   case 0x88: // movb a, r
  779   case 0x89: // movl a, r
  780   case 0x8A: // movb r, a
  781   case 0x8B: // movl r, a
  782   case 0x8F: // popl a
  783     debug_only(has_disp32 = true);
  784     break;
  785 
  786   case 0x68: // pushq #32
  787     if (which == end_pc_operand) {
  788       return ip + 4;
  789     }
  790     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  791     return ip;                  // not produced by emit_operand
  792 
  793   case 0x66: // movw ... (size prefix)
  794     again_after_size_prefix2:
  795     switch (0xFF & *ip++) {
  796     case REX:
  797     case REX_B:
  798     case REX_X:
  799     case REX_XB:
  800     case REX_R:
  801     case REX_RB:
  802     case REX_RX:
  803     case REX_RXB:
  804     case REX_W:
  805     case REX_WB:
  806     case REX_WX:
  807     case REX_WXB:
  808     case REX_WR:
  809     case REX_WRB:
  810     case REX_WRX:
  811     case REX_WRXB:
  812       NOT_LP64(assert(false, "64bit prefix found"));
  813       goto again_after_size_prefix2;
  814     case 0x8B: // movw r, a
  815     case 0x89: // movw a, r
  816       debug_only(has_disp32 = true);
  817       break;
  818     case 0xC7: // movw a, #16
  819       debug_only(has_disp32 = true);
  820       tail_size = 2;  // the imm16
  821       break;
  822     case 0x0F: // several SSE/SSE2 variants
  823       ip--;    // reparse the 0x0F
  824       goto again_after_prefix;
  825     default:
  826       ShouldNotReachHere();
  827     }
  828     break;
  829 
  830   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  831     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  832     // these asserts are somewhat nonsensical
  833 #ifndef _LP64
  834     assert(which == imm_operand || which == disp32_operand,
  835            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  836 #else
  837     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  838            which == narrow_oop_operand && !is_64bit,
  839            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  840 #endif // _LP64
  841     return ip;
  842 
  843   case 0x69: // imul r, a, #32
  844   case 0xC7: // movl a, #32(oop?)
  845     tail_size = 4;
  846     debug_only(has_disp32 = true); // has both kinds of operands!
  847     break;
  848 
  849   case 0x0F: // movx..., etc.
  850     switch (0xFF & *ip++) {
  851     case 0x3A: // pcmpestri
  852       tail_size = 1;
  853     case 0x38: // ptest, pmovzxbw
  854       ip++; // skip opcode
  855       debug_only(has_disp32 = true); // has both kinds of operands!
  856       break;
  857 
  858     case 0x70: // pshufd r, r/a, #8
  859       debug_only(has_disp32 = true); // has both kinds of operands!
  860     case 0x73: // psrldq r, #8
  861       tail_size = 1;
  862       break;
  863 
  864     case 0x12: // movlps
  865     case 0x28: // movaps
  866     case 0x2E: // ucomiss
  867     case 0x2F: // comiss
  868     case 0x54: // andps
  869     case 0x55: // andnps
  870     case 0x56: // orps
  871     case 0x57: // xorps
  872     case 0x58: // addpd
  873     case 0x59: // mulpd
  874     case 0x6E: // movd
  875     case 0x7E: // movd
  876     case 0x6F: // movdq
  877     case 0x7F: // movdq
  878     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  879     case 0xFE: // paddd
  880       debug_only(has_disp32 = true);
  881       break;
  882 
  883     case 0xAD: // shrd r, a, %cl
  884     case 0xAF: // imul r, a
  885     case 0xBE: // movsbl r, a (movsxb)
  886     case 0xBF: // movswl r, a (movsxw)
  887     case 0xB6: // movzbl r, a (movzxb)
  888     case 0xB7: // movzwl r, a (movzxw)
  889     case REP16(0x40): // cmovl cc, r, a
  890     case 0xB0: // cmpxchgb
  891     case 0xB1: // cmpxchg
  892     case 0xC1: // xaddl
  893     case 0xC7: // cmpxchg8
  894     case REP16(0x90): // setcc a
  895       debug_only(has_disp32 = true);
  896       // fall out of the switch to decode the address
  897       break;
  898 
  899     case 0xC4: // pinsrw r, a, #8
  900       debug_only(has_disp32 = true);
  901     case 0xC5: // pextrw r, r, #8
  902       tail_size = 1;  // the imm8
  903       break;
  904 
  905     case 0xAC: // shrd r, a, #8
  906       debug_only(has_disp32 = true);
  907       tail_size = 1;  // the imm8
  908       break;
  909 
  910     case REP16(0x80): // jcc rdisp32
  911       if (which == end_pc_operand)  return ip + 4;
  912       assert(which == call32_operand, "jcc has no disp32 or imm");
  913       return ip;
  914     default:
  915       ShouldNotReachHere();
  916     }
  917     break;
  918 
  919   case 0x81: // addl a, #32; addl r, #32
  920     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  921     // on 32bit in the case of cmpl, the imm might be an oop
  922     tail_size = 4;
  923     debug_only(has_disp32 = true); // has both kinds of operands!
  924     break;
  925 
  926   case 0x83: // addl a, #8; addl r, #8
  927     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  928     debug_only(has_disp32 = true); // has both kinds of operands!
  929     tail_size = 1;
  930     break;
  931 
  932   case 0x9B:
  933     switch (0xFF & *ip++) {
  934     case 0xD9: // fnstcw a
  935       debug_only(has_disp32 = true);
  936       break;
  937     default:
  938       ShouldNotReachHere();
  939     }
  940     break;
  941 
  942   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
  943   case REP4(0x10): // adc...
  944   case REP4(0x20): // and...
  945   case REP4(0x30): // xor...
  946   case REP4(0x08): // or...
  947   case REP4(0x18): // sbb...
  948   case REP4(0x28): // sub...
  949   case 0xF7: // mull a
  950   case 0x8D: // lea r, a
  951   case 0x87: // xchg r, a
  952   case REP4(0x38): // cmp...
  953   case 0x85: // test r, a
  954     debug_only(has_disp32 = true); // has both kinds of operands!
  955     break;
  956 
  957   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
  958   case 0xC6: // movb a, #8
  959   case 0x80: // cmpb a, #8
  960   case 0x6B: // imul r, a, #8
  961     debug_only(has_disp32 = true); // has both kinds of operands!
  962     tail_size = 1; // the imm8
  963     break;
  964 
  965   case 0xC4: // VEX_3bytes
  966   case 0xC5: // VEX_2bytes
  967     assert((UseAVX > 0), "shouldn't have VEX prefix");
  968     assert(ip == inst+1, "no prefixes allowed");
  969     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
  970     // but they have prefix 0x0F and processed when 0x0F processed above.
  971     //
  972     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
  973     // instructions (these instructions are not supported in 64-bit mode).
  974     // To distinguish them bits [7:6] are set in the VEX second byte since
  975     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
  976     // those VEX bits REX and vvvv bits are inverted.
  977     //
  978     // Fortunately C2 doesn't generate these instructions so we don't need
  979     // to check for them in product version.
  980 
  981     // Check second byte
  982     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
  983 
  984     int vex_opcode;
  985     // First byte
  986     if ((0xFF & *inst) == VEX_3bytes) {
  987       vex_opcode = VEX_OPCODE_MASK & *ip;
  988       ip++; // third byte
  989       is_64bit = ((VEX_W & *ip) == VEX_W);
  990     } else {
  991       vex_opcode = VEX_OPCODE_0F;
  992     }
  993     ip++; // opcode
  994     // To find the end of instruction (which == end_pc_operand).
  995     switch (vex_opcode) {
  996       case VEX_OPCODE_0F:
  997         switch (0xFF & *ip) {
  998         case 0x70: // pshufd r, r/a, #8
  999         case 0x71: // ps[rl|ra|ll]w r, #8
 1000         case 0x72: // ps[rl|ra|ll]d r, #8
 1001         case 0x73: // ps[rl|ra|ll]q r, #8
 1002         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1003         case 0xC4: // pinsrw r, r, r/a, #8
 1004         case 0xC5: // pextrw r/a, r, #8
 1005         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1006           tail_size = 1;  // the imm8
 1007           break;
 1008         }
 1009         break;
 1010       case VEX_OPCODE_0F_3A:
 1011         tail_size = 1;
 1012         break;
 1013     }
 1014     ip++; // skip opcode
 1015     debug_only(has_disp32 = true); // has both kinds of operands!
 1016     break;
 1017 
 1018   case 0x62: // EVEX_4bytes
 1019     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1020     assert(ip == inst+1, "no prefixes allowed");
 1021     // no EVEX collisions, all instructions that have 0x62 opcodes
 1022     // have EVEX versions and are subopcodes of 0x66
 1023     ip++; // skip P0 and exmaine W in P1
 1024     is_64bit = ((VEX_W & *ip) == VEX_W);
 1025     ip++; // move to P2
 1026     ip++; // skip P2, move to opcode
 1027     // To find the end of instruction (which == end_pc_operand).
 1028     switch (0xFF & *ip) {
 1029     case 0x22: // pinsrd r, r/a, #8
 1030     case 0x61: // pcmpestri r, r/a, #8
 1031     case 0x70: // pshufd r, r/a, #8
 1032     case 0x73: // psrldq r, #8
 1033     case 0x1f: // evpcmpd/evpcmpq
 1034     case 0x3f: // evpcmpb/evpcmpw
 1035       tail_size = 1;  // the imm8
 1036       break;
 1037     default:
 1038       break;
 1039     }
 1040     ip++; // skip opcode
 1041     debug_only(has_disp32 = true); // has both kinds of operands!
 1042     break;
 1043 
 1044   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1045   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1046   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1047   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1048   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1049   case 0xDF: // fild_d a; fistp_d a
 1050   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1051   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1052   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1053     debug_only(has_disp32 = true);
 1054     break;
 1055 
 1056   case 0xE8: // call rdisp32
 1057   case 0xE9: // jmp  rdisp32
 1058     if (which == end_pc_operand)  return ip + 4;
 1059     assert(which == call32_operand, "call has no disp32 or imm");
 1060     return ip;
 1061 
 1062   case 0xF0:                    // Lock
 1063     goto again_after_prefix;
 1064 
 1065   case 0xF3:                    // For SSE
 1066   case 0xF2:                    // For SSE2
 1067     switch (0xFF & *ip++) {
 1068     case REX:
 1069     case REX_B:
 1070     case REX_X:
 1071     case REX_XB:
 1072     case REX_R:
 1073     case REX_RB:
 1074     case REX_RX:
 1075     case REX_RXB:
 1076     case REX_W:
 1077     case REX_WB:
 1078     case REX_WX:
 1079     case REX_WXB:
 1080     case REX_WR:
 1081     case REX_WRB:
 1082     case REX_WRX:
 1083     case REX_WRXB:
 1084       NOT_LP64(assert(false, "found 64bit prefix"));
 1085       ip++;
 1086     default:
 1087       ip++;
 1088     }
 1089     debug_only(has_disp32 = true); // has both kinds of operands!
 1090     break;
 1091 
 1092   default:
 1093     ShouldNotReachHere();
 1094 
 1095 #undef REP8
 1096 #undef REP16
 1097   }
 1098 
 1099   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1100 #ifdef _LP64
 1101   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1102 #else
 1103   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1104   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1105 #endif // LP64
 1106   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1107 
 1108   // parse the output of emit_operand
 1109   int op2 = 0xFF & *ip++;
 1110   int base = op2 & 0x07;
 1111   int op3 = -1;
 1112   const int b100 = 4;
 1113   const int b101 = 5;
 1114   if (base == b100 && (op2 >> 6) != 3) {
 1115     op3 = 0xFF & *ip++;
 1116     base = op3 & 0x07;   // refetch the base
 1117   }
 1118   // now ip points at the disp (if any)
 1119 
 1120   switch (op2 >> 6) {
 1121   case 0:
 1122     // [00 reg  100][ss index base]
 1123     // [00 reg  100][00   100  esp]
 1124     // [00 reg base]
 1125     // [00 reg  100][ss index  101][disp32]
 1126     // [00 reg  101]               [disp32]
 1127 
 1128     if (base == b101) {
 1129       if (which == disp32_operand)
 1130         return ip;              // caller wants the disp32
 1131       ip += 4;                  // skip the disp32
 1132     }
 1133     break;
 1134 
 1135   case 1:
 1136     // [01 reg  100][ss index base][disp8]
 1137     // [01 reg  100][00   100  esp][disp8]
 1138     // [01 reg base]               [disp8]
 1139     ip += 1;                    // skip the disp8
 1140     break;
 1141 
 1142   case 2:
 1143     // [10 reg  100][ss index base][disp32]
 1144     // [10 reg  100][00   100  esp][disp32]
 1145     // [10 reg base]               [disp32]
 1146     if (which == disp32_operand)
 1147       return ip;                // caller wants the disp32
 1148     ip += 4;                    // skip the disp32
 1149     break;
 1150 
 1151   case 3:
 1152     // [11 reg base]  (not a memory addressing mode)
 1153     break;
 1154   }
 1155 
 1156   if (which == end_pc_operand) {
 1157     return ip + tail_size;
 1158   }
 1159 
 1160 #ifdef _LP64
 1161   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1162 #else
 1163   assert(which == imm_operand, "instruction has only an imm field");
 1164 #endif // LP64
 1165   return ip;
 1166 }
 1167 
 1168 address Assembler::locate_next_instruction(address inst) {
 1169   // Secretly share code with locate_operand:
 1170   return locate_operand(inst, end_pc_operand);
 1171 }
 1172 
 1173 
 1174 #ifdef ASSERT
 1175 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1176   address inst = inst_mark();
 1177   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 1178   address opnd;
 1179 
 1180   Relocation* r = rspec.reloc();
 1181   if (r->type() == relocInfo::none) {
 1182     return;
 1183   } else if (r->is_call() || format == call32_operand) {
 1184     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1185     opnd = locate_operand(inst, call32_operand);
 1186   } else if (r->is_data()) {
 1187     assert(format == imm_operand || format == disp32_operand
 1188            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1189     opnd = locate_operand(inst, (WhichOperand)format);
 1190   } else {
 1191     assert(format == imm_operand, "cannot specify a format");
 1192     return;
 1193   }
 1194   assert(opnd == pc(), "must put operand where relocs can find it");
 1195 }
 1196 #endif // ASSERT
 1197 
 1198 void Assembler::emit_operand(Register reg, Address adr,
 1199                              int rip_relative_correction) {
 1200   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1201                adr._rspec,
 1202                rip_relative_correction);
 1203 }
 1204 
 1205 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 1206     if (adr.isxmmindex()) {
 1207        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
 1208     } else {
 1209        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 1210        adr._rspec);
 1211     }
 1212 }
 1213 
 1214 // Now the Assembler instructions (identical for 32/64 bits)
 1215 
 1216 void Assembler::adcl(Address dst, int32_t imm32) {
 1217   InstructionMark im(this);
 1218   prefix(dst);
 1219   emit_arith_operand(0x81, rdx, dst, imm32);
 1220 }
 1221 
 1222 void Assembler::adcl(Address dst, Register src) {
 1223   InstructionMark im(this);
 1224   prefix(dst, src);
 1225   emit_int8(0x11);
 1226   emit_operand(src, dst);
 1227 }
 1228 
 1229 void Assembler::adcl(Register dst, int32_t imm32) {
 1230   prefix(dst);
 1231   emit_arith(0x81, 0xD0, dst, imm32);
 1232 }
 1233 
 1234 void Assembler::adcl(Register dst, Address src) {
 1235   InstructionMark im(this);
 1236   prefix(src, dst);
 1237   emit_int8(0x13);
 1238   emit_operand(dst, src);
 1239 }
 1240 
 1241 void Assembler::adcl(Register dst, Register src) {
 1242   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1243   emit_arith(0x13, 0xC0, dst, src);
 1244 }
 1245 
 1246 void Assembler::addl(Address dst, int32_t imm32) {
 1247   InstructionMark im(this);
 1248   prefix(dst);
 1249   emit_arith_operand(0x81, rax, dst, imm32);
 1250 }
 1251 
 1252 void Assembler::addb(Address dst, int imm8) {
 1253   InstructionMark im(this);
 1254   prefix(dst);
 1255   emit_int8((unsigned char)0x80);
 1256   emit_operand(rax, dst, 1);
 1257   emit_int8(imm8);
 1258 }
 1259 
 1260 void Assembler::addw(Register dst, Register src) {
 1261   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1262   emit_arith(0x03, 0xC0, dst, src);
 1263 }
 1264 
 1265 void Assembler::addw(Address dst, int imm16) {
 1266   InstructionMark im(this);
 1267   emit_int8(0x66);
 1268   prefix(dst);
 1269   emit_int8((unsigned char)0x81);
 1270   emit_operand(rax, dst, 2);
 1271   emit_int16(imm16);
 1272 }
 1273 
 1274 void Assembler::addl(Address dst, Register src) {
 1275   InstructionMark im(this);
 1276   prefix(dst, src);
 1277   emit_int8(0x01);
 1278   emit_operand(src, dst);
 1279 }
 1280 
 1281 void Assembler::addl(Register dst, int32_t imm32) {
 1282   prefix(dst);
 1283   emit_arith(0x81, 0xC0, dst, imm32);
 1284 }
 1285 
 1286 void Assembler::addl(Register dst, Address src) {
 1287   InstructionMark im(this);
 1288   prefix(src, dst);
 1289   emit_int8(0x03);
 1290   emit_operand(dst, src);
 1291 }
 1292 
 1293 void Assembler::addl(Register dst, Register src) {
 1294   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1295   emit_arith(0x03, 0xC0, dst, src);
 1296 }
 1297 
 1298 void Assembler::addr_nop_4() {
 1299   assert(UseAddressNop, "no CPU support");
 1300   // 4 bytes: NOP DWORD PTR [EAX+0]
 1301   emit_int32(0x0F,
 1302              0x1F,
 1303              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1304              0);   // 8-bits offset (1 byte)
 1305 }
 1306 
 1307 void Assembler::addr_nop_5() {
 1308   assert(UseAddressNop, "no CPU support");
 1309   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1310   emit_int32(0x0F,
 1311              0x1F,
 1312              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1313              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1314   emit_int8(0);     // 8-bits offset (1 byte)
 1315 }
 1316 
 1317 void Assembler::addr_nop_7() {
 1318   assert(UseAddressNop, "no CPU support");
 1319   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1320   emit_int24(0x0F,
 1321              0x1F,
 1322              (unsigned char)0x80);
 1323                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1324   emit_int32(0);   // 32-bits offset (4 bytes)
 1325 }
 1326 
 1327 void Assembler::addr_nop_8() {
 1328   assert(UseAddressNop, "no CPU support");
 1329   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1330   emit_int32(0x0F,
 1331              0x1F,
 1332              (unsigned char)0x84,
 1333                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1334              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1335   emit_int32(0);    // 32-bits offset (4 bytes)
 1336 }
 1337 
 1338 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1341   attributes.set_rex_vex_w_reverted();
 1342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1343   emit_int16(0x58, (0xC0 | encode));
 1344 }
 1345 
 1346 void Assembler::addsd(XMMRegister dst, Address src) {
 1347   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1348   InstructionMark im(this);
 1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1350   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1351   attributes.set_rex_vex_w_reverted();
 1352   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1353   emit_int8(0x58);
 1354   emit_operand(dst, src);
 1355 }
 1356 
 1357 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1358   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1360   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1361   emit_int16(0x58, (0xC0 | encode));
 1362 }
 1363 
 1364 void Assembler::addss(XMMRegister dst, Address src) {
 1365   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1366   InstructionMark im(this);
 1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1368   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1369   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1370   emit_int8(0x58);
 1371   emit_operand(dst, src);
 1372 }
 1373 
 1374 void Assembler::aesdec(XMMRegister dst, Address src) {
 1375   assert(VM_Version::supports_aes(), "");
 1376   InstructionMark im(this);
 1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1378   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1379   emit_int8((unsigned char)0xDE);
 1380   emit_operand(dst, src);
 1381 }
 1382 
 1383 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1384   assert(VM_Version::supports_aes(), "");
 1385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1386   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1387   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1388 }
 1389 
 1390 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1391   assert(VM_Version::supports_avx512_vaes(), "");
 1392   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1393   attributes.set_is_evex_instruction();
 1394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1395   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1396 }
 1397 
 1398 
 1399 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1400   assert(VM_Version::supports_aes(), "");
 1401   InstructionMark im(this);
 1402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1403   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1404   emit_int8((unsigned char)0xDF);
 1405   emit_operand(dst, src);
 1406 }
 1407 
 1408 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1409   assert(VM_Version::supports_aes(), "");
 1410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1411   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1412   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1413 }
 1414 
 1415 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1416   assert(VM_Version::supports_avx512_vaes(), "");
 1417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1418   attributes.set_is_evex_instruction();
 1419   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1420   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1421 }
 1422 
 1423 void Assembler::aesenc(XMMRegister dst, Address src) {
 1424   assert(VM_Version::supports_aes(), "");
 1425   InstructionMark im(this);
 1426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1427   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1428   emit_int8((unsigned char)0xDC);
 1429   emit_operand(dst, src);
 1430 }
 1431 
 1432 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1433   assert(VM_Version::supports_aes(), "");
 1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1435   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1436   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1437 }
 1438 
 1439 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1440   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1442   attributes.set_is_evex_instruction();
 1443   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1444   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1445 }
 1446 
 1447 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1448   assert(VM_Version::supports_aes(), "");
 1449   InstructionMark im(this);
 1450   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1451   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1452   emit_int8((unsigned char)0xDD);
 1453   emit_operand(dst, src);
 1454 }
 1455 
 1456 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1457   assert(VM_Version::supports_aes(), "");
 1458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1460   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1461 }
 1462 
 1463 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1464   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1465   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1466   attributes.set_is_evex_instruction();
 1467   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1468   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1469 }
 1470 
 1471 void Assembler::andb(Address dst, Register src) {
 1472   InstructionMark im(this);
 1473   prefix(dst, src, true);
 1474   emit_int8(0x20);
 1475   emit_operand(src, dst);
 1476 }
 1477 
 1478 void Assembler::andw(Register dst, Register src) {
 1479   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1480   emit_arith(0x23, 0xC0, dst, src);
 1481 }
 1482 
 1483 void Assembler::andl(Address dst, int32_t imm32) {
 1484   InstructionMark im(this);
 1485   prefix(dst);
 1486   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1487 }
 1488 
 1489 void Assembler::andl(Register dst, int32_t imm32) {
 1490   prefix(dst);
 1491   emit_arith(0x81, 0xE0, dst, imm32);
 1492 }
 1493 
 1494 void Assembler::andl(Address dst, Register src) {
 1495   InstructionMark im(this);
 1496   prefix(dst, src);
 1497   emit_int8(0x21);
 1498   emit_operand(src, dst);
 1499 }
 1500 
 1501 void Assembler::andl(Register dst, Address src) {
 1502   InstructionMark im(this);
 1503   prefix(src, dst);
 1504   emit_int8(0x23);
 1505   emit_operand(dst, src);
 1506 }
 1507 
 1508 void Assembler::andl(Register dst, Register src) {
 1509   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1510   emit_arith(0x23, 0xC0, dst, src);
 1511 }
 1512 
 1513 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1514   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1516   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1517   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1518 }
 1519 
 1520 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1521   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1522   InstructionMark im(this);
 1523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1524   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1525   emit_int8((unsigned char)0xF2);
 1526   emit_operand(dst, src2);
 1527 }
 1528 
 1529 void Assembler::bsfl(Register dst, Register src) {
 1530   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1531   emit_int24(0x0F,
 1532              (unsigned char)0xBC,
 1533              0xC0 | encode);
 1534 }
 1535 
 1536 void Assembler::bsrl(Register dst, Register src) {
 1537   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1538   emit_int24(0x0F,
 1539              (unsigned char)0xBD,
 1540              0xC0 | encode);
 1541 }
 1542 
 1543 void Assembler::bswapl(Register reg) { // bswap
 1544   int encode = prefix_and_encode(reg->encoding());
 1545   emit_int16(0x0F, (0xC8 | encode));
 1546 }
 1547 
 1548 void Assembler::blsil(Register dst, Register src) {
 1549   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1550   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1551   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1553 }
 1554 
 1555 void Assembler::blsil(Register dst, Address src) {
 1556   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1557   InstructionMark im(this);
 1558   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1559   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1560   emit_int8((unsigned char)0xF3);
 1561   emit_operand(rbx, src);
 1562 }
 1563 
 1564 void Assembler::blsmskl(Register dst, Register src) {
 1565   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1566   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1567   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1568   emit_int16((unsigned char)0xF3,
 1569              0xC0 | encode);
 1570 }
 1571 
 1572 void Assembler::blsmskl(Register dst, Address src) {
 1573   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1574   InstructionMark im(this);
 1575   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1576   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1577   emit_int8((unsigned char)0xF3);
 1578   emit_operand(rdx, src);
 1579 }
 1580 
 1581 void Assembler::blsrl(Register dst, Register src) {
 1582   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1583   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1584   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1585   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1586 }
 1587 
 1588 void Assembler::blsrl(Register dst, Address src) {
 1589   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1590   InstructionMark im(this);
 1591   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1592   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1593   emit_int8((unsigned char)0xF3);
 1594   emit_operand(rcx, src);
 1595 }
 1596 
 1597 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1598   // suspect disp32 is always good
 1599   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1600 
 1601   if (L.is_bound()) {
 1602     const int long_size = 5;
 1603     int offs = (int)( target(L) - pc() );
 1604     assert(offs <= 0, "assembler error");
 1605     InstructionMark im(this);
 1606     // 1110 1000 #32-bit disp
 1607     emit_int8((unsigned char)0xE8);
 1608     emit_data(offs - long_size, rtype, operand);
 1609   } else {
 1610     InstructionMark im(this);
 1611     // 1110 1000 #32-bit disp
 1612     L.add_patch_at(code(), locator());
 1613 
 1614     emit_int8((unsigned char)0xE8);
 1615     emit_data(int(0), rtype, operand);
 1616   }
 1617 }
 1618 
 1619 void Assembler::call(Register dst) {
 1620   int encode = prefix_and_encode(dst->encoding());
 1621   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1622 }
 1623 
 1624 
 1625 void Assembler::call(Address adr) {
 1626   InstructionMark im(this);
 1627   prefix(adr);
 1628   emit_int8((unsigned char)0xFF);
 1629   emit_operand(rdx, adr);
 1630 }
 1631 
 1632 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1633   InstructionMark im(this);
 1634   emit_int8((unsigned char)0xE8);
 1635   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1636   // Entry is NULL in case of a scratch emit.
 1637   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1638   // Technically, should use call32_operand, but this format is
 1639   // implied by the fact that we're emitting a call instruction.
 1640 
 1641   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1642   emit_data((int) disp, rspec, operand);
 1643 }
 1644 
 1645 void Assembler::cdql() {
 1646   emit_int8((unsigned char)0x99);
 1647 }
 1648 
 1649 void Assembler::cld() {
 1650   emit_int8((unsigned char)0xFC);
 1651 }
 1652 
 1653 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1654   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1655   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1656   emit_int24(0x0F,
 1657              0x40 | cc,
 1658              0xC0 | encode);
 1659 }
 1660 
 1661 
 1662 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1663   InstructionMark im(this);
 1664   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1665   prefix(src, dst);
 1666   emit_int16(0x0F, (0x40 | cc));
 1667   emit_operand(dst, src);
 1668 }
 1669 
 1670 void Assembler::cmpb(Address dst, int imm8) {
 1671   InstructionMark im(this);
 1672   prefix(dst);
 1673   emit_int8((unsigned char)0x80);
 1674   emit_operand(rdi, dst, 1);
 1675   emit_int8(imm8);
 1676 }
 1677 
 1678 void Assembler::cmpl(Address dst, int32_t imm32) {
 1679   InstructionMark im(this);
 1680   prefix(dst);
 1681   emit_int8((unsigned char)0x81);
 1682   emit_operand(rdi, dst, 4);
 1683   emit_int32(imm32);
 1684 }
 1685 
 1686 void Assembler::cmp(Register dst, int32_t imm32) {
 1687   prefix(dst);
 1688   emit_int8((unsigned char)0x3D);
 1689   emit_int32(imm32);
 1690 }
 1691 
 1692 void Assembler::cmpl(Register dst, int32_t imm32) {
 1693   prefix(dst);
 1694   emit_arith(0x81, 0xF8, dst, imm32);
 1695 }
 1696 
 1697 void Assembler::cmpl(Register dst, Register src) {
 1698   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1699   emit_arith(0x3B, 0xC0, dst, src);
 1700 }
 1701 
 1702 void Assembler::cmpl(Register dst, Address  src) {
 1703   InstructionMark im(this);
 1704   prefix(src, dst);
 1705   emit_int8(0x3B);
 1706   emit_operand(dst, src);
 1707 }
 1708 
 1709 void Assembler::cmpw(Address dst, int imm16) {
 1710   InstructionMark im(this);
 1711   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
 1712   emit_int16(0x66, (unsigned char)0x81);
 1713   emit_operand(rdi, dst, 2);
 1714   emit_int16(imm16);
 1715 }
 1716 
 1717 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1718 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1719 // The ZF is set if the compared values were equal, and cleared otherwise.
 1720 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1721   InstructionMark im(this);
 1722   prefix(adr, reg);
 1723   emit_int16(0x0F, (unsigned char)0xB1);
 1724   emit_operand(reg, adr);
 1725 }
 1726 
 1727 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1728   InstructionMark im(this);
 1729   size_prefix();
 1730   prefix(adr, reg);
 1731   emit_int16(0x0F, (unsigned char)0xB1);
 1732   emit_operand(reg, adr);
 1733 }
 1734 
 1735 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1736 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1737 // The ZF is set if the compared values were equal, and cleared otherwise.
 1738 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1739   InstructionMark im(this);
 1740   prefix(adr, reg, true);
 1741   emit_int16(0x0F, (unsigned char)0xB0);
 1742   emit_operand(reg, adr);
 1743 }
 1744 
 1745 void Assembler::comisd(XMMRegister dst, Address src) {
 1746   // NOTE: dbx seems to decode this as comiss even though the
 1747   // 0x66 is there. Strangly ucomisd comes out correct
 1748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1749   InstructionMark im(this);
 1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1751   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1752   attributes.set_rex_vex_w_reverted();
 1753   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1754   emit_int8(0x2F);
 1755   emit_operand(dst, src);
 1756 }
 1757 
 1758 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1760   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1761   attributes.set_rex_vex_w_reverted();
 1762   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1763   emit_int16(0x2F, (0xC0 | encode));
 1764 }
 1765 
 1766 void Assembler::comiss(XMMRegister dst, Address src) {
 1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1768   InstructionMark im(this);
 1769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1770   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1771   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1772   emit_int8(0x2F);
 1773   emit_operand(dst, src);
 1774 }
 1775 
 1776 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1777   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1779   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1780   emit_int16(0x2F, (0xC0 | encode));
 1781 }
 1782 
 1783 void Assembler::cpuid() {
 1784   emit_int16(0x0F, (unsigned char)0xA2);
 1785 }
 1786 
 1787 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1788 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1789 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1790 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1791 //
 1792 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1793 //
 1794 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1795 //
 1796 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1797 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1798   assert(VM_Version::supports_sse4_2(), "");
 1799   int8_t w = 0x01;
 1800   Prefix p = Prefix_EMPTY;
 1801 
 1802   emit_int8((unsigned char)0xF2);
 1803   switch (sizeInBytes) {
 1804   case 1:
 1805     w = 0;
 1806     break;
 1807   case 2:
 1808   case 4:
 1809     break;
 1810   LP64_ONLY(case 8:)
 1811     // This instruction is not valid in 32 bits
 1812     // Note:
 1813     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1814     //
 1815     // Page B - 72   Vol. 2C says
 1816     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1817     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1818     //                                                                            F0!!!
 1819     // while 3 - 208 Vol. 2A
 1820     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1821     //
 1822     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1823     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1824     p = REX_W;
 1825     break;
 1826   default:
 1827     assert(0, "Unsupported value for a sizeInBytes argument");
 1828     break;
 1829   }
 1830   LP64_ONLY(prefix(crc, v, p);)
 1831   emit_int32(0x0F,
 1832              0x38,
 1833              0xF0 | w,
 1834              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1835 }
 1836 
 1837 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1838   assert(VM_Version::supports_sse4_2(), "");
 1839   InstructionMark im(this);
 1840   int8_t w = 0x01;
 1841   Prefix p = Prefix_EMPTY;
 1842 
 1843   emit_int8((int8_t)0xF2);
 1844   switch (sizeInBytes) {
 1845   case 1:
 1846     w = 0;
 1847     break;
 1848   case 2:
 1849   case 4:
 1850     break;
 1851   LP64_ONLY(case 8:)
 1852     // This instruction is not valid in 32 bits
 1853     p = REX_W;
 1854     break;
 1855   default:
 1856     assert(0, "Unsupported value for a sizeInBytes argument");
 1857     break;
 1858   }
 1859   LP64_ONLY(prefix(crc, adr, p);)
 1860   emit_int24(0x0F, 0x38, (0xF0 | w));
 1861   emit_operand(crc, adr);
 1862 }
 1863 
 1864 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1868   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1869 }
 1870 
 1871 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1872   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1873   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1875   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1876 }
 1877 
 1878 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 1879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1882   emit_int16(0x5B, (0xC0 | encode));
 1883 }
 1884 
 1885 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1886   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1887   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1889   emit_int16(0x5B, (0xC0 | encode));
 1890 }
 1891 
 1892 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1893   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1894   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1895   attributes.set_rex_vex_w_reverted();
 1896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1897   emit_int16(0x5A, (0xC0 | encode));
 1898 }
 1899 
 1900 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 1901   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1902   InstructionMark im(this);
 1903   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1904   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1905   attributes.set_rex_vex_w_reverted();
 1906   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1907   emit_int8(0x5A);
 1908   emit_operand(dst, src);
 1909 }
 1910 
 1911 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1913   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1914   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1915   emit_int16(0x2A, (0xC0 | encode));
 1916 }
 1917 
 1918 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1919   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1920   InstructionMark im(this);
 1921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1922   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1923   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1924   emit_int8(0x2A);
 1925   emit_operand(dst, src);
 1926 }
 1927 
 1928 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1929   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1931   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1932   emit_int16(0x2A, (0xC0 | encode));
 1933 }
 1934 
 1935 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1937   InstructionMark im(this);
 1938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1939   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1940   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1941   emit_int8(0x2A);
 1942   emit_operand(dst, src);
 1943 }
 1944 
 1945 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1946   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1947   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1948   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1949   emit_int16(0x2A, (0xC0 | encode));
 1950 }
 1951 
 1952 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1953   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1955   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1956   emit_int16(0x5A, (0xC0 | encode));
 1957 }
 1958 
 1959 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 1960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1961   InstructionMark im(this);
 1962   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1963   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1964   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1965   emit_int8(0x5A);
 1966   emit_operand(dst, src);
 1967 }
 1968 
 1969 
 1970 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 1971   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1973   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1974   emit_int16(0x2C, (0xC0 | encode));
 1975 }
 1976 
 1977 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 1978   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1980   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1981   emit_int16(0x2C, (0xC0 | encode));
 1982 }
 1983 
 1984 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 1985   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1986   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 1987   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1988   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1989   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1990 }
 1991 
 1992 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 1993   assert(VM_Version::supports_ssse3(), "");
 1994   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 1995   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1996   emit_int16(0x1C, (0xC0 | encode));
 1997 }
 1998 
 1999 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2000   assert(VM_Version::supports_ssse3(), "");
 2001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2002   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2003   emit_int16(0x1D, (0xC0 | encode));
 2004 }
 2005 
 2006 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2007   assert(VM_Version::supports_ssse3(), "");
 2008   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2009   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2010   emit_int16(0x1E, (0xC0 | encode));
 2011 }
 2012 
 2013 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2014   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2015          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2016          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2017   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2018   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2019   emit_int16(0x1C, (0xC0 | encode));
 2020 }
 2021 
 2022 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2023   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2024          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2025          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2026   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2027   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2028   emit_int16(0x1D, (0xC0 | encode));
 2029 }
 2030 
 2031 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2032   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2033   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2034   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2035   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2036   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2037   emit_int16(0x1E, (0xC0 | encode));
 2038 }
 2039 
 2040 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2041   assert(UseAVX > 2, "");
 2042   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2043   attributes.set_is_evex_instruction();
 2044   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2045   emit_int16(0x1F, (0xC0 | encode));
 2046 }
 2047 
 2048 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2049   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2050   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2051   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2052   emit_int16(0x5A, (0xC0 | encode));
 2053 }
 2054 
 2055 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2056   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2057   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2058   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2059   attributes.set_rex_vex_w_reverted();
 2060   emit_int16(0x5A, (0xC0 | encode));
 2061 }
 2062 
 2063 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2064   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2065   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2066   attributes.set_is_evex_instruction();
 2067   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2068   emit_int16(0x5B, (0xC0 | encode));
 2069 }
 2070 
 2071 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2072   assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
 2073   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2074   attributes.set_is_evex_instruction();
 2075   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2076   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2077 }
 2078 
 2079 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2080   assert(UseAVX > 2  && VM_Version::supports_avx512bw(), "");
 2081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2082   attributes.set_is_evex_instruction();
 2083   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2084   emit_int16(0x30, (0xC0 | encode));
 2085 }
 2086 
 2087 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2088   assert(UseAVX > 2, "");
 2089   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2090   attributes.set_is_evex_instruction();
 2091   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2092   emit_int16(0x33, (0xC0 | encode));
 2093 }
 2094 
 2095 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2096   assert(UseAVX > 2, "");
 2097   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2098   attributes.set_is_evex_instruction();
 2099   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2100   emit_int16(0x31, (0xC0 | encode));
 2101 }
 2102 
 2103 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2104   assert(UseAVX > 2, "");
 2105   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2106   attributes.set_is_evex_instruction();
 2107   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2108   emit_int16(0x35, (0xC0 | encode));
 2109 }
 2110 
 2111 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2112   assert(UseAVX > 2, "");
 2113   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2114   attributes.set_is_evex_instruction();
 2115   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2116   emit_int16(0x32, (0xC0 | encode));
 2117 }
 2118 
 2119 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2120   assert(UseAVX > 2, "");
 2121   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2122   attributes.set_is_evex_instruction();
 2123   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2124   emit_int16(0x34, (0xC0 | encode));
 2125 }
 2126 
 2127 void Assembler::decl(Address dst) {
 2128   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2129   InstructionMark im(this);
 2130   prefix(dst);
 2131   emit_int8((unsigned char)0xFF);
 2132   emit_operand(rcx, dst);
 2133 }
 2134 
 2135 void Assembler::divsd(XMMRegister dst, Address src) {
 2136   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2137   InstructionMark im(this);
 2138   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2139   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2140   attributes.set_rex_vex_w_reverted();
 2141   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2142   emit_int8(0x5E);
 2143   emit_operand(dst, src);
 2144 }
 2145 
 2146 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2147   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2148   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2149   attributes.set_rex_vex_w_reverted();
 2150   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2151   emit_int16(0x5E, (0xC0 | encode));
 2152 }
 2153 
 2154 void Assembler::divss(XMMRegister dst, Address src) {
 2155   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2156   InstructionMark im(this);
 2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2158   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2159   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2160   emit_int8(0x5E);
 2161   emit_operand(dst, src);
 2162 }
 2163 
 2164 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2168   emit_int16(0x5E, (0xC0 | encode));
 2169 }
 2170 
 2171 void Assembler::hlt() {
 2172   emit_int8((unsigned char)0xF4);
 2173 }
 2174 
 2175 void Assembler::idivl(Register src) {
 2176   int encode = prefix_and_encode(src->encoding());
 2177   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2178 }
 2179 
 2180 void Assembler::divl(Register src) { // Unsigned
 2181   int encode = prefix_and_encode(src->encoding());
 2182   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2183 }
 2184 
 2185 void Assembler::imull(Register src) {
 2186   int encode = prefix_and_encode(src->encoding());
 2187   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2188 }
 2189 
 2190 void Assembler::imull(Register dst, Register src) {
 2191   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2192   emit_int24(0x0F,
 2193              (unsigned char)0xAF,
 2194              (0xC0 | encode));
 2195 }
 2196 
 2197 void Assembler::imull(Register dst, Address src, int32_t value) {
 2198   InstructionMark im(this);
 2199   prefix(src, dst);
 2200   if (is8bit(value)) {
 2201     emit_int8((unsigned char)0x6B);
 2202     emit_operand(dst, src);
 2203     emit_int8(value);
 2204   } else {
 2205     emit_int8((unsigned char)0x69);
 2206     emit_operand(dst, src);
 2207     emit_int32(value);
 2208   }
 2209 }
 2210 
 2211 void Assembler::imull(Register dst, Register src, int value) {
 2212   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2213   if (is8bit(value)) {
 2214     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2215   } else {
 2216     emit_int16(0x69, (0xC0 | encode));
 2217     emit_int32(value);
 2218   }
 2219 }
 2220 
 2221 void Assembler::imull(Register dst, Address src) {
 2222   InstructionMark im(this);
 2223   prefix(src, dst);
 2224   emit_int16(0x0F, (unsigned char)0xAF);
 2225   emit_operand(dst, src);
 2226 }
 2227 
 2228 
 2229 void Assembler::incl(Address dst) {
 2230   // Don't use it directly. Use MacroAssembler::increment() instead.
 2231   InstructionMark im(this);
 2232   prefix(dst);
 2233   emit_int8((unsigned char)0xFF);
 2234   emit_operand(rax, dst);
 2235 }
 2236 
 2237 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2238   InstructionMark im(this);
 2239   assert((0 <= cc) && (cc < 16), "illegal cc");
 2240   if (L.is_bound()) {
 2241     address dst = target(L);
 2242     assert(dst != NULL, "jcc most probably wrong");
 2243 
 2244     const int short_size = 2;
 2245     const int long_size = 6;
 2246     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
 2247     if (maybe_short && is8bit(offs - short_size)) {
 2248       // 0111 tttn #8-bit disp
 2249       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2250     } else {
 2251       // 0000 1111 1000 tttn #32-bit disp
 2252       assert(is_simm32(offs - long_size),
 2253              "must be 32bit offset (call4)");
 2254       emit_int16(0x0F, (0x80 | cc));
 2255       emit_int32(offs - long_size);
 2256     }
 2257   } else {
 2258     // Note: could eliminate cond. jumps to this jump if condition
 2259     //       is the same however, seems to be rather unlikely case.
 2260     // Note: use jccb() if label to be bound is very close to get
 2261     //       an 8-bit displacement
 2262     L.add_patch_at(code(), locator());
 2263     emit_int16(0x0F, (0x80 | cc));
 2264     emit_int32(0);
 2265   }
 2266 }
 2267 
 2268 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2269   if (L.is_bound()) {
 2270     const int short_size = 2;
 2271     address entry = target(L);
 2272 #ifdef ASSERT
 2273     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2274     intptr_t delta = short_branch_delta();
 2275     if (delta != 0) {
 2276       dist += (dist < 0 ? (-delta) :delta);
 2277     }
 2278     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2279 #endif
 2280     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
 2281     // 0111 tttn #8-bit disp
 2282     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2283   } else {
 2284     InstructionMark im(this);
 2285     L.add_patch_at(code(), locator(), file, line);
 2286     emit_int16(0x70 | cc, 0);
 2287   }
 2288 }
 2289 
 2290 void Assembler::jmp(Address adr) {
 2291   InstructionMark im(this);
 2292   prefix(adr);
 2293   emit_int8((unsigned char)0xFF);
 2294   emit_operand(rsp, adr);
 2295 }
 2296 
 2297 void Assembler::jmp(Label& L, bool maybe_short) {
 2298   if (L.is_bound()) {
 2299     address entry = target(L);
 2300     assert(entry != NULL, "jmp most probably wrong");
 2301     InstructionMark im(this);
 2302     const int short_size = 2;
 2303     const int long_size = 5;
 2304     intptr_t offs = entry - pc();
 2305     if (maybe_short && is8bit(offs - short_size)) {
 2306       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2307     } else {
 2308       emit_int8((unsigned char)0xE9);
 2309       emit_int32(offs - long_size);
 2310     }
 2311   } else {
 2312     // By default, forward jumps are always 32-bit displacements, since
 2313     // we can't yet know where the label will be bound.  If you're sure that
 2314     // the forward jump will not run beyond 256 bytes, use jmpb to
 2315     // force an 8-bit displacement.
 2316     InstructionMark im(this);
 2317     L.add_patch_at(code(), locator());
 2318     emit_int8((unsigned char)0xE9);
 2319     emit_int32(0);
 2320   }
 2321 }
 2322 
 2323 void Assembler::jmp(Register entry) {
 2324   int encode = prefix_and_encode(entry->encoding());
 2325   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2326 }
 2327 
 2328 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2329   InstructionMark im(this);
 2330   emit_int8((unsigned char)0xE9);
 2331   assert(dest != NULL, "must have a target");
 2332   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2333   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2334   emit_data(disp, rspec.reloc(), call32_operand);
 2335 }
 2336 
 2337 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2338   if (L.is_bound()) {
 2339     const int short_size = 2;
 2340     address entry = target(L);
 2341     assert(entry != NULL, "jmp most probably wrong");
 2342 #ifdef ASSERT
 2343     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
 2344     intptr_t delta = short_branch_delta();
 2345     if (delta != 0) {
 2346       dist += (dist < 0 ? (-delta) :delta);
 2347     }
 2348     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
 2349 #endif
 2350     intptr_t offs = entry - pc();
 2351     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2352   } else {
 2353     InstructionMark im(this);
 2354     L.add_patch_at(code(), locator(), file, line);
 2355     emit_int16((unsigned char)0xEB, 0);
 2356   }
 2357 }
 2358 
 2359 void Assembler::ldmxcsr( Address src) {
 2360   if (UseAVX > 0 ) {
 2361     InstructionMark im(this);
 2362     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2363     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2364     emit_int8((unsigned char)0xAE);
 2365     emit_operand(as_Register(2), src);
 2366   } else {
 2367     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2368     InstructionMark im(this);
 2369     prefix(src);
 2370     emit_int16(0x0F, (unsigned char)0xAE);
 2371     emit_operand(as_Register(2), src);
 2372   }
 2373 }
 2374 
 2375 void Assembler::leal(Register dst, Address src) {
 2376   InstructionMark im(this);
 2377 #ifdef _LP64
 2378   emit_int8(0x67); // addr32
 2379   prefix(src, dst);
 2380 #endif // LP64
 2381   emit_int8((unsigned char)0x8D);
 2382   emit_operand(dst, src);
 2383 }
 2384 
 2385 void Assembler::lfence() {
 2386   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2387 }
 2388 
 2389 void Assembler::lock() {
 2390   emit_int8((unsigned char)0xF0);
 2391 }
 2392 
 2393 void Assembler::size_prefix() {
 2394   emit_int8(0x66);
 2395 }
 2396 
 2397 void Assembler::lzcntl(Register dst, Register src) {
 2398   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2399   emit_int8((unsigned char)0xF3);
 2400   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2401   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2402 }
 2403 
 2404 // Emit mfence instruction
 2405 void Assembler::mfence() {
 2406   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2407   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2408 }
 2409 
 2410 // Emit sfence instruction
 2411 void Assembler::sfence() {
 2412   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2413   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2414 }
 2415 
 2416 void Assembler::mov(Register dst, Register src) {
 2417   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2418 }
 2419 
 2420 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2422   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2423   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2424   attributes.set_rex_vex_w_reverted();
 2425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2426   emit_int16(0x28, (0xC0 | encode));
 2427 }
 2428 
 2429 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2430   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2431   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2432   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2433   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2434   emit_int16(0x28, (0xC0 | encode));
 2435 }
 2436 
 2437 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2438   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2440   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2441   emit_int16(0x16, (0xC0 | encode));
 2442 }
 2443 
 2444 void Assembler::movb(Register dst, Address src) {
 2445   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2446   InstructionMark im(this);
 2447   prefix(src, dst, true);
 2448   emit_int8((unsigned char)0x8A);
 2449   emit_operand(dst, src);
 2450 }
 2451 
 2452 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2453   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2454   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2455   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2456   attributes.set_rex_vex_w_reverted();
 2457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2458   emit_int16(0x12, 0xC0 | encode);
 2459 }
 2460 
 2461 void Assembler::kmovbl(KRegister dst, Register src) {
 2462   assert(VM_Version::supports_avx512dq(), "");
 2463   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2465   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2466 }
 2467 
 2468 void Assembler::kmovbl(Register dst, KRegister src) {
 2469   assert(VM_Version::supports_avx512dq(), "");
 2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2472   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2473 }
 2474 
 2475 void Assembler::kmovwl(KRegister dst, Register src) {
 2476   assert(VM_Version::supports_evex(), "");
 2477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2478   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2479   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2480 }
 2481 
 2482 void Assembler::kmovwl(Register dst, KRegister src) {
 2483   assert(VM_Version::supports_evex(), "");
 2484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2485   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2486   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2487 }
 2488 
 2489 void Assembler::kmovwl(KRegister dst, Address src) {
 2490   assert(VM_Version::supports_evex(), "");
 2491   InstructionMark im(this);
 2492   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2493   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2494   emit_int8((unsigned char)0x90);
 2495   emit_operand((Register)dst, src);
 2496 }
 2497 
 2498 void Assembler::kmovwl(Address dst, KRegister src) {
 2499   assert(VM_Version::supports_evex(), "");
 2500   InstructionMark im(this);
 2501   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2502   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2503   emit_int8((unsigned char)0x91);
 2504   emit_operand((Register)src, dst);
 2505 }
 2506 
 2507 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2508   assert(VM_Version::supports_avx512bw(), "");
 2509   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2510   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2511   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2512 }
 2513 
 2514 void Assembler::kmovdl(KRegister dst, Register src) {
 2515   assert(VM_Version::supports_avx512bw(), "");
 2516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2517   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2518   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2519 }
 2520 
 2521 void Assembler::kmovdl(Register dst, KRegister src) {
 2522   assert(VM_Version::supports_avx512bw(), "");
 2523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2524   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2525   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2526 }
 2527 
 2528 void Assembler::kmovql(KRegister dst, KRegister src) {
 2529   assert(VM_Version::supports_avx512bw(), "");
 2530   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2531   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2532   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2533 }
 2534 
 2535 void Assembler::kmovql(KRegister dst, Address src) {
 2536   assert(VM_Version::supports_avx512bw(), "");
 2537   InstructionMark im(this);
 2538   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2539   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2540   emit_int8((unsigned char)0x90);
 2541   emit_operand((Register)dst, src);
 2542 }
 2543 
 2544 void Assembler::kmovql(Address dst, KRegister src) {
 2545   assert(VM_Version::supports_avx512bw(), "");
 2546   InstructionMark im(this);
 2547   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2548   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2549   emit_int8((unsigned char)0x91);
 2550   emit_operand((Register)src, dst);
 2551 }
 2552 
 2553 void Assembler::kmovql(KRegister dst, Register src) {
 2554   assert(VM_Version::supports_avx512bw(), "");
 2555   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2556   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2557   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2558 }
 2559 
 2560 void Assembler::kmovql(Register dst, KRegister src) {
 2561   assert(VM_Version::supports_avx512bw(), "");
 2562   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2563   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2564   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2565 }
 2566 
 2567 void Assembler::knotwl(KRegister dst, KRegister src) {
 2568   assert(VM_Version::supports_evex(), "");
 2569   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2570   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2571   emit_int16(0x44, (0xC0 | encode));
 2572 }
 2573 
 2574 void Assembler::knotql(KRegister dst, KRegister src) {
 2575   assert(VM_Version::supports_avx512bw(), "");
 2576   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2578   emit_int16(0x44, (0xC0 | encode));
 2579 }
 2580 
 2581 // This instruction produces ZF or CF flags
 2582 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2583   assert(VM_Version::supports_avx512dq(), "");
 2584   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2585   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2586   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2587 }
 2588 
 2589 // This instruction produces ZF or CF flags
 2590 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2591   assert(VM_Version::supports_evex(), "");
 2592   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2593   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2594   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2595 }
 2596 
 2597 // This instruction produces ZF or CF flags
 2598 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2599   assert(VM_Version::supports_avx512bw(), "");
 2600   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2601   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2602   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2603 }
 2604 
 2605 // This instruction produces ZF or CF flags
 2606 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2607   assert(VM_Version::supports_avx512bw(), "");
 2608   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2609   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2610   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2611 }
 2612 
 2613 // This instruction produces ZF or CF flags
 2614 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2615   assert(VM_Version::supports_avx512bw(), "");
 2616   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2617   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2618   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2619 }
 2620 
 2621 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2622   assert(VM_Version::supports_avx512bw(), "");
 2623   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2624   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2625   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2626 }
 2627 
 2628 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2629   assert(VM_Version::supports_avx512bw(), "");
 2630   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2631   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2632   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2633 }
 2634 
 2635 void Assembler::movb(Address dst, int imm8) {
 2636   InstructionMark im(this);
 2637    prefix(dst);
 2638   emit_int8((unsigned char)0xC6);
 2639   emit_operand(rax, dst, 1);
 2640   emit_int8(imm8);
 2641 }
 2642 
 2643 
 2644 void Assembler::movb(Address dst, Register src) {
 2645   assert(src->has_byte_register(), "must have byte register");
 2646   InstructionMark im(this);
 2647   prefix(dst, src, true);
 2648   emit_int8((unsigned char)0x88);
 2649   emit_operand(src, dst);
 2650 }
 2651 
 2652 void Assembler::movdl(XMMRegister dst, Register src) {
 2653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2655   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2656   emit_int16(0x6E, (0xC0 | encode));
 2657 }
 2658 
 2659 void Assembler::movdl(Register dst, XMMRegister src) {
 2660   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2661   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2662   // swap src/dst to get correct prefix
 2663   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2664   emit_int16(0x7E, (0xC0 | encode));
 2665 }
 2666 
 2667 void Assembler::movdl(XMMRegister dst, Address src) {
 2668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2669   InstructionMark im(this);
 2670   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2671   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2672   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2673   emit_int8(0x6E);
 2674   emit_operand(dst, src);
 2675 }
 2676 
 2677 void Assembler::movdl(Address dst, XMMRegister src) {
 2678   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2679   InstructionMark im(this);
 2680   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2681   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2682   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2683   emit_int8(0x7E);
 2684   emit_operand(src, dst);
 2685 }
 2686 
 2687 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 2688   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2690   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2691   emit_int16(0x6F, (0xC0 | encode));
 2692 }
 2693 
 2694 void Assembler::movdqa(XMMRegister dst, Address src) {
 2695   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2696   InstructionMark im(this);
 2697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2698   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2699   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2700   emit_int8(0x6F);
 2701   emit_operand(dst, src);
 2702 }
 2703 
 2704 void Assembler::movdqu(XMMRegister dst, Address src) {
 2705   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2706   InstructionMark im(this);
 2707   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2708   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2709   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2710   emit_int8(0x6F);
 2711   emit_operand(dst, src);
 2712 }
 2713 
 2714 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 2715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2718   emit_int16(0x6F, (0xC0 | encode));
 2719 }
 2720 
 2721 void Assembler::movdqu(Address dst, XMMRegister src) {
 2722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2723   InstructionMark im(this);
 2724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2725   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2726   attributes.reset_is_clear_context();
 2727   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2728   emit_int8(0x7F);
 2729   emit_operand(src, dst);
 2730 }
 2731 
 2732 // Move Unaligned 256bit Vector
 2733 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2734   assert(UseAVX > 0, "");
 2735   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2736   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2737   emit_int16(0x6F, (0xC0 | encode));
 2738 }
 2739 
 2740 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 2741   assert(UseAVX > 0, "");
 2742   InstructionMark im(this);
 2743   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2744   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2745   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2746   emit_int8(0x6F);
 2747   emit_operand(dst, src);
 2748 }
 2749 
 2750 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 2751   assert(UseAVX > 0, "");
 2752   InstructionMark im(this);
 2753   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2754   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2755   attributes.reset_is_clear_context();
 2756   // swap src<->dst for encoding
 2757   assert(src != xnoreg, "sanity");
 2758   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2759   emit_int8(0x7F);
 2760   emit_operand(src, dst);
 2761 }
 2762 
 2763 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 2764 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 2765   assert(VM_Version::supports_evex(), "");
 2766   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2767   attributes.set_is_evex_instruction();
 2768   if (merge) {
 2769     attributes.reset_is_clear_context();
 2770   }
 2771   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2772   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2773   emit_int16(0x6F, (0xC0 | encode));
 2774 }
 2775 
 2776 void Assembler::evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) {
 2777   assert(VM_Version::supports_evex(), "");
 2778   InstructionMark im(this);
 2779   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2780   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2781   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2782   attributes.set_is_evex_instruction();
 2783   if (merge) {
 2784     attributes.reset_is_clear_context();
 2785   }
 2786   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2787   emit_int8(0x6F);
 2788   emit_operand(dst, src);
 2789 }
 2790 
 2791 void Assembler::evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) {
 2792   assert(VM_Version::supports_evex(), "");
 2793   assert(src != xnoreg, "sanity");
 2794   InstructionMark im(this);
 2795   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2796   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2797   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2798   attributes.set_is_evex_instruction();
 2799   if (merge) {
 2800     attributes.reset_is_clear_context();
 2801   }
 2802   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2803   emit_int8(0x7F);
 2804   emit_operand(src, dst);
 2805 }
 2806 
 2807 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2808   assert(VM_Version::supports_avx512vlbw(), "");
 2809   InstructionMark im(this);
 2810   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2811   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2812   attributes.set_embedded_opmask_register_specifier(mask);
 2813   attributes.set_is_evex_instruction();
 2814   if (merge) {
 2815     attributes.reset_is_clear_context();
 2816   }
 2817   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2818   emit_int8(0x6F);
 2819   emit_operand(dst, src);
 2820 }
 2821 
 2822 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2823   assert(VM_Version::supports_avx512vlbw(), "");
 2824   assert(src != xnoreg, "sanity");
 2825   InstructionMark im(this);
 2826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2827   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2828   attributes.set_embedded_opmask_register_specifier(mask);
 2829   attributes.set_is_evex_instruction();
 2830   if (merge) {
 2831     attributes.reset_is_clear_context();
 2832   }
 2833   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2834   emit_int8(0x7F);
 2835   emit_operand(src, dst);
 2836 }
 2837 
 2838 void Assembler::evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) {
 2839   assert(VM_Version::supports_evex(), "");
 2840   InstructionMark im(this);
 2841   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2842   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2843   attributes.set_is_evex_instruction();
 2844   if (merge) {
 2845     attributes.reset_is_clear_context();
 2846   }
 2847   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2848   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2849   emit_int8(0x6F);
 2850   emit_operand(dst, src);
 2851 }
 2852 
 2853 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2854   assert(VM_Version::supports_avx512vlbw(), "");
 2855   InstructionMark im(this);
 2856   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2857   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2858   attributes.set_embedded_opmask_register_specifier(mask);
 2859   attributes.set_is_evex_instruction();
 2860   if (merge) {
 2861     attributes.reset_is_clear_context();
 2862   }
 2863   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2864   emit_int8(0x6F);
 2865   emit_operand(dst, src);
 2866 }
 2867 
 2868 void Assembler::evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) {
 2869   assert(VM_Version::supports_evex(), "");
 2870   assert(src != xnoreg, "sanity");
 2871   InstructionMark im(this);
 2872   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2873   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2874   attributes.set_is_evex_instruction();
 2875   if (merge) {
 2876     attributes.reset_is_clear_context();
 2877   }
 2878   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
 2879   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
 2880   emit_int8(0x7F);
 2881   emit_operand(src, dst);
 2882 }
 2883 
 2884 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2885   assert(VM_Version::supports_avx512vlbw(), "");
 2886   assert(src != xnoreg, "sanity");
 2887   InstructionMark im(this);
 2888   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2889   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2890   attributes.set_embedded_opmask_register_specifier(mask);
 2891   attributes.set_is_evex_instruction();
 2892   if (merge) {
 2893     attributes.reset_is_clear_context();
 2894   }
 2895   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2896   emit_int8(0x7F);
 2897   emit_operand(src, dst);
 2898 }
 2899 
 2900 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 2901   // Unmasked instruction
 2902   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 2903 }
 2904 
 2905 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2906   assert(VM_Version::supports_evex(), "");
 2907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2908   attributes.set_embedded_opmask_register_specifier(mask);
 2909   attributes.set_is_evex_instruction();
 2910   if (merge) {
 2911     attributes.reset_is_clear_context();
 2912   }
 2913   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2914   emit_int16(0x6F, (0xC0 | encode));
 2915 }
 2916 
 2917 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 2918   // Unmasked instruction
 2919   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 2920 }
 2921 
 2922 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2923   assert(VM_Version::supports_evex(), "");
 2924   InstructionMark im(this);
 2925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 2926   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2927   attributes.set_embedded_opmask_register_specifier(mask);
 2928   attributes.set_is_evex_instruction();
 2929   if (merge) {
 2930     attributes.reset_is_clear_context();
 2931   }
 2932   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2933   emit_int8(0x6F);
 2934   emit_operand(dst, src);
 2935 }
 2936 
 2937 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 2938   // Unmasked isntruction
 2939   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 2940 }
 2941 
 2942 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2943   assert(VM_Version::supports_evex(), "");
 2944   assert(src != xnoreg, "sanity");
 2945   InstructionMark im(this);
 2946   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2947   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2948   attributes.set_embedded_opmask_register_specifier(mask);
 2949   attributes.set_is_evex_instruction();
 2950   if (merge) {
 2951     attributes.reset_is_clear_context();
 2952   }
 2953   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2954   emit_int8(0x7F);
 2955   emit_operand(src, dst);
 2956 }
 2957 
 2958 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 2959   // Unmasked instruction
 2960   if (dst->encoding() == src->encoding()) return;
 2961   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 2962 }
 2963 
 2964 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 2965   assert(VM_Version::supports_evex(), "");
 2966   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2967   attributes.set_embedded_opmask_register_specifier(mask);
 2968   attributes.set_is_evex_instruction();
 2969   if (merge) {
 2970     attributes.reset_is_clear_context();
 2971   }
 2972   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2973   emit_int16(0x6F, (0xC0 | encode));
 2974 }
 2975 
 2976 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 2977   // Unmasked instruction
 2978   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 2979 }
 2980 
 2981 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 2982   assert(VM_Version::supports_evex(), "");
 2983   InstructionMark im(this);
 2984   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 2985   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 2986   attributes.set_embedded_opmask_register_specifier(mask);
 2987   attributes.set_is_evex_instruction();
 2988   if (merge) {
 2989     attributes.reset_is_clear_context();
 2990   }
 2991   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2992   emit_int8(0x6F);
 2993   emit_operand(dst, src);
 2994 }
 2995 
 2996 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 2997   // Unmasked instruction
 2998   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 2999 }
 3000 
 3001 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3002   assert(VM_Version::supports_evex(), "");
 3003   assert(src != xnoreg, "sanity");
 3004   InstructionMark im(this);
 3005   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3006   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3007   attributes.set_embedded_opmask_register_specifier(mask);
 3008   if (merge) {
 3009     attributes.reset_is_clear_context();
 3010   }
 3011   attributes.set_is_evex_instruction();
 3012   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3013   emit_int8(0x7F);
 3014   emit_operand(src, dst);
 3015 }
 3016 
 3017 // Uses zero extension on 64bit
 3018 
 3019 void Assembler::movl(Register dst, int32_t imm32) {
 3020   int encode = prefix_and_encode(dst->encoding());
 3021   emit_int8(0xB8 | encode);
 3022   emit_int32(imm32);
 3023 }
 3024 
 3025 void Assembler::movl(Register dst, Register src) {
 3026   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3027   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3028 }
 3029 
 3030 void Assembler::movl(Register dst, Address src) {
 3031   InstructionMark im(this);
 3032   prefix(src, dst);
 3033   emit_int8((unsigned char)0x8B);
 3034   emit_operand(dst, src);
 3035 }
 3036 
 3037 void Assembler::movl(Address dst, int32_t imm32) {
 3038   InstructionMark im(this);
 3039   prefix(dst);
 3040   emit_int8((unsigned char)0xC7);
 3041   emit_operand(rax, dst, 4);
 3042   emit_int32(imm32);
 3043 }
 3044 
 3045 void Assembler::movl(Address dst, Register src) {
 3046   InstructionMark im(this);
 3047   prefix(dst, src);
 3048   emit_int8((unsigned char)0x89);
 3049   emit_operand(src, dst);
 3050 }
 3051 
 3052 // New cpus require to use movsd and movss to avoid partial register stall
 3053 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3054 // The selection is done in MacroAssembler::movdbl() and movflt().
 3055 void Assembler::movlpd(XMMRegister dst, Address src) {
 3056   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3057   InstructionMark im(this);
 3058   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3059   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3060   attributes.set_rex_vex_w_reverted();
 3061   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3062   emit_int8(0x12);
 3063   emit_operand(dst, src);
 3064 }
 3065 
 3066 void Assembler::movq(XMMRegister dst, Address src) {
 3067   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3068   InstructionMark im(this);
 3069   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3070   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3071   attributes.set_rex_vex_w_reverted();
 3072   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3073   emit_int8(0x7E);
 3074   emit_operand(dst, src);
 3075 }
 3076 
 3077 void Assembler::movq(Address dst, XMMRegister src) {
 3078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3079   InstructionMark im(this);
 3080   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3081   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3082   attributes.set_rex_vex_w_reverted();
 3083   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3084   emit_int8((unsigned char)0xD6);
 3085   emit_operand(src, dst);
 3086 }
 3087 
 3088 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3089   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3090   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3091   attributes.set_rex_vex_w_reverted();
 3092   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3093   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3094 }
 3095 
 3096 void Assembler::movq(Register dst, XMMRegister src) {
 3097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3098   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3099   // swap src/dst to get correct prefix
 3100   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3101   emit_int16(0x7E, (0xC0 | encode));
 3102 }
 3103 
 3104 void Assembler::movq(XMMRegister dst, Register src) {
 3105   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3106   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3107   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3108   emit_int16(0x6E, (0xC0 | encode));
 3109 }
 3110 
 3111 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3112   InstructionMark im(this);
 3113   prefix(src, dst);
 3114   emit_int16(0x0F, (unsigned char)0xBE);
 3115   emit_operand(dst, src);
 3116 }
 3117 
 3118 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3119   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3120   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3121   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3122 }
 3123 
 3124 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3125   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3126   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3127   attributes.set_rex_vex_w_reverted();
 3128   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3129   emit_int16(0x10, (0xC0 | encode));
 3130 }
 3131 
 3132 void Assembler::movsd(XMMRegister dst, Address src) {
 3133   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3134   InstructionMark im(this);
 3135   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3136   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3137   attributes.set_rex_vex_w_reverted();
 3138   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3139   emit_int8(0x10);
 3140   emit_operand(dst, src);
 3141 }
 3142 
 3143 void Assembler::movsd(Address dst, XMMRegister src) {
 3144   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3145   InstructionMark im(this);
 3146   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3147   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3148   attributes.reset_is_clear_context();
 3149   attributes.set_rex_vex_w_reverted();
 3150   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3151   emit_int8(0x11);
 3152   emit_operand(src, dst);
 3153 }
 3154 
 3155 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3156   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3158   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3159   emit_int16(0x10, (0xC0 | encode));
 3160 }
 3161 
 3162 void Assembler::movss(XMMRegister dst, Address src) {
 3163   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3164   InstructionMark im(this);
 3165   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3166   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3167   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3168   emit_int8(0x10);
 3169   emit_operand(dst, src);
 3170 }
 3171 
 3172 void Assembler::movss(Address dst, XMMRegister src) {
 3173   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3174   InstructionMark im(this);
 3175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3176   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3177   attributes.reset_is_clear_context();
 3178   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3179   emit_int8(0x11);
 3180   emit_operand(src, dst);
 3181 }
 3182 
 3183 void Assembler::movswl(Register dst, Address src) { // movsxw
 3184   InstructionMark im(this);
 3185   prefix(src, dst);
 3186   emit_int16(0x0F, (unsigned char)0xBF);
 3187   emit_operand(dst, src);
 3188 }
 3189 
 3190 void Assembler::movswl(Register dst, Register src) { // movsxw
 3191   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3192   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3193 }
 3194 
 3195 void Assembler::movw(Address dst, int imm16) {
 3196   InstructionMark im(this);
 3197 
 3198   emit_int8(0x66); // switch to 16-bit mode
 3199   prefix(dst);
 3200   emit_int8((unsigned char)0xC7);
 3201   emit_operand(rax, dst, 2);
 3202   emit_int16(imm16);
 3203 }
 3204 
 3205 void Assembler::movw(Register dst, Address src) {
 3206   InstructionMark im(this);
 3207   emit_int8(0x66);
 3208   prefix(src, dst);
 3209   emit_int8((unsigned char)0x8B);
 3210   emit_operand(dst, src);
 3211 }
 3212 
 3213 void Assembler::movw(Address dst, Register src) {
 3214   InstructionMark im(this);
 3215   emit_int8(0x66);
 3216   prefix(dst, src);
 3217   emit_int8((unsigned char)0x89);
 3218   emit_operand(src, dst);
 3219 }
 3220 
 3221 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3222   InstructionMark im(this);
 3223   prefix(src, dst);
 3224   emit_int16(0x0F, (unsigned char)0xB6);
 3225   emit_operand(dst, src);
 3226 }
 3227 
 3228 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3229   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3230   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3231   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3232 }
 3233 
 3234 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3235   InstructionMark im(this);
 3236   prefix(src, dst);
 3237   emit_int16(0x0F, (unsigned char)0xB7);
 3238   emit_operand(dst, src);
 3239 }
 3240 
 3241 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3242   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3243   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3244 }
 3245 
 3246 void Assembler::mull(Address src) {
 3247   InstructionMark im(this);
 3248   prefix(src);
 3249   emit_int8((unsigned char)0xF7);
 3250   emit_operand(rsp, src);
 3251 }
 3252 
 3253 void Assembler::mull(Register src) {
 3254   int encode = prefix_and_encode(src->encoding());
 3255   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3256 }
 3257 
 3258 void Assembler::mulsd(XMMRegister dst, Address src) {
 3259   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3260   InstructionMark im(this);
 3261   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3262   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3263   attributes.set_rex_vex_w_reverted();
 3264   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3265   emit_int8(0x59);
 3266   emit_operand(dst, src);
 3267 }
 3268 
 3269 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3271   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3272   attributes.set_rex_vex_w_reverted();
 3273   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3274   emit_int16(0x59, (0xC0 | encode));
 3275 }
 3276 
 3277 void Assembler::mulss(XMMRegister dst, Address src) {
 3278   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3279   InstructionMark im(this);
 3280   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3281   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3282   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3283   emit_int8(0x59);
 3284   emit_operand(dst, src);
 3285 }
 3286 
 3287 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3288   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3291   emit_int16(0x59, (0xC0 | encode));
 3292 }
 3293 
 3294 void Assembler::negl(Register dst) {
 3295   int encode = prefix_and_encode(dst->encoding());
 3296   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3297 }
 3298 
 3299 void Assembler::negl(Address dst) {
 3300   InstructionMark im(this);
 3301   prefix(dst);
 3302   emit_int8((unsigned char)0xF7);
 3303   emit_operand(as_Register(3), dst);
 3304 }
 3305 
 3306 void Assembler::nop(int i) {
 3307 #ifdef ASSERT
 3308   assert(i > 0, " ");
 3309   // The fancy nops aren't currently recognized by debuggers making it a
 3310   // pain to disassemble code while debugging. If asserts are on clearly
 3311   // speed is not an issue so simply use the single byte traditional nop
 3312   // to do alignment.
 3313 
 3314   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3315   return;
 3316 
 3317 #endif // ASSERT
 3318 
 3319   if (UseAddressNop && VM_Version::is_intel()) {
 3320     //
 3321     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3322     //  1: 0x90
 3323     //  2: 0x66 0x90
 3324     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3325     //  4: 0x0F 0x1F 0x40 0x00
 3326     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3327     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3328     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3329     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3330     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3331     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3332     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3333 
 3334     // The rest coding is Intel specific - don't use consecutive address nops
 3335 
 3336     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3337     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3338     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3339     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3340 
 3341     while(i >= 15) {
 3342       // For Intel don't generate consecutive addess nops (mix with regular nops)
 3343       i -= 15;
 3344       emit_int24(0x66, 0x66, 0x66);
 3345       addr_nop_8();
 3346       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3347     }
 3348     switch (i) {
 3349       case 14:
 3350         emit_int8(0x66); // size prefix
 3351       case 13:
 3352         emit_int8(0x66); // size prefix
 3353       case 12:
 3354         addr_nop_8();
 3355         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3356         break;
 3357       case 11:
 3358         emit_int8(0x66); // size prefix
 3359       case 10:
 3360         emit_int8(0x66); // size prefix
 3361       case 9:
 3362         emit_int8(0x66); // size prefix
 3363       case 8:
 3364         addr_nop_8();
 3365         break;
 3366       case 7:
 3367         addr_nop_7();
 3368         break;
 3369       case 6:
 3370         emit_int8(0x66); // size prefix
 3371       case 5:
 3372         addr_nop_5();
 3373         break;
 3374       case 4:
 3375         addr_nop_4();
 3376         break;
 3377       case 3:
 3378         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3379         emit_int8(0x66); // size prefix
 3380       case 2:
 3381         emit_int8(0x66); // size prefix
 3382       case 1:
 3383         emit_int8((unsigned char)0x90);
 3384                          // nop
 3385         break;
 3386       default:
 3387         assert(i == 0, " ");
 3388     }
 3389     return;
 3390   }
 3391   if (UseAddressNop && VM_Version::is_amd_family()) {
 3392     //
 3393     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3394     //  1: 0x90
 3395     //  2: 0x66 0x90
 3396     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3397     //  4: 0x0F 0x1F 0x40 0x00
 3398     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3399     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3400     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3401     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3402     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3403     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3404     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3405 
 3406     // The rest coding is AMD specific - use consecutive address nops
 3407 
 3408     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3409     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3410     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3411     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3412     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3413     //     Size prefixes (0x66) are added for larger sizes
 3414 
 3415     while(i >= 22) {
 3416       i -= 11;
 3417       emit_int24(0x66, 0x66, 0x66);
 3418       addr_nop_8();
 3419     }
 3420     // Generate first nop for size between 21-12
 3421     switch (i) {
 3422       case 21:
 3423         i -= 1;
 3424         emit_int8(0x66); // size prefix
 3425       case 20:
 3426       case 19:
 3427         i -= 1;
 3428         emit_int8(0x66); // size prefix
 3429       case 18:
 3430       case 17:
 3431         i -= 1;
 3432         emit_int8(0x66); // size prefix
 3433       case 16:
 3434       case 15:
 3435         i -= 8;
 3436         addr_nop_8();
 3437         break;
 3438       case 14:
 3439       case 13:
 3440         i -= 7;
 3441         addr_nop_7();
 3442         break;
 3443       case 12:
 3444         i -= 6;
 3445         emit_int8(0x66); // size prefix
 3446         addr_nop_5();
 3447         break;
 3448       default:
 3449         assert(i < 12, " ");
 3450     }
 3451 
 3452     // Generate second nop for size between 11-1
 3453     switch (i) {
 3454       case 11:
 3455         emit_int8(0x66); // size prefix
 3456       case 10:
 3457         emit_int8(0x66); // size prefix
 3458       case 9:
 3459         emit_int8(0x66); // size prefix
 3460       case 8:
 3461         addr_nop_8();
 3462         break;
 3463       case 7:
 3464         addr_nop_7();
 3465         break;
 3466       case 6:
 3467         emit_int8(0x66); // size prefix
 3468       case 5:
 3469         addr_nop_5();
 3470         break;
 3471       case 4:
 3472         addr_nop_4();
 3473         break;
 3474       case 3:
 3475         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3476         emit_int8(0x66); // size prefix
 3477       case 2:
 3478         emit_int8(0x66); // size prefix
 3479       case 1:
 3480         emit_int8((unsigned char)0x90);
 3481                          // nop
 3482         break;
 3483       default:
 3484         assert(i == 0, " ");
 3485     }
 3486     return;
 3487   }
 3488 
 3489   if (UseAddressNop && VM_Version::is_zx()) {
 3490     //
 3491     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3492     //  1: 0x90
 3493     //  2: 0x66 0x90
 3494     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3495     //  4: 0x0F 0x1F 0x40 0x00
 3496     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3497     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3498     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3499     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3500     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3501     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3502     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3503 
 3504     // The rest coding is ZX specific - don't use consecutive address nops
 3505 
 3506     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3507     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3508     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3509     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3510 
 3511     while (i >= 15) {
 3512       // For ZX don't generate consecutive addess nops (mix with regular nops)
 3513       i -= 15;
 3514       emit_int24(0x66, 0x66, 0x66);
 3515       addr_nop_8();
 3516       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3517     }
 3518     switch (i) {
 3519       case 14:
 3520         emit_int8(0x66); // size prefix
 3521       case 13:
 3522         emit_int8(0x66); // size prefix
 3523       case 12:
 3524         addr_nop_8();
 3525         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3526         break;
 3527       case 11:
 3528         emit_int8(0x66); // size prefix
 3529       case 10:
 3530         emit_int8(0x66); // size prefix
 3531       case 9:
 3532         emit_int8(0x66); // size prefix
 3533       case 8:
 3534         addr_nop_8();
 3535         break;
 3536       case 7:
 3537         addr_nop_7();
 3538         break;
 3539       case 6:
 3540         emit_int8(0x66); // size prefix
 3541       case 5:
 3542         addr_nop_5();
 3543         break;
 3544       case 4:
 3545         addr_nop_4();
 3546         break;
 3547       case 3:
 3548         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3549         emit_int8(0x66); // size prefix
 3550       case 2:
 3551         emit_int8(0x66); // size prefix
 3552       case 1:
 3553         emit_int8((unsigned char)0x90);
 3554                          // nop
 3555         break;
 3556       default:
 3557         assert(i == 0, " ");
 3558     }
 3559     return;
 3560   }
 3561 
 3562   // Using nops with size prefixes "0x66 0x90".
 3563   // From AMD Optimization Guide:
 3564   //  1: 0x90
 3565   //  2: 0x66 0x90
 3566   //  3: 0x66 0x66 0x90
 3567   //  4: 0x66 0x66 0x66 0x90
 3568   //  5: 0x66 0x66 0x90 0x66 0x90
 3569   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 3570   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 3571   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 3572   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3573   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 3574   //
 3575   while (i > 12) {
 3576     i -= 4;
 3577     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3578   }
 3579   // 1 - 12 nops
 3580   if (i > 8) {
 3581     if (i > 9) {
 3582       i -= 1;
 3583       emit_int8(0x66);
 3584     }
 3585     i -= 3;
 3586     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3587   }
 3588   // 1 - 8 nops
 3589   if (i > 4) {
 3590     if (i > 6) {
 3591       i -= 1;
 3592       emit_int8(0x66);
 3593     }
 3594     i -= 3;
 3595     emit_int24(0x66, 0x66, (unsigned char)0x90);
 3596   }
 3597   switch (i) {
 3598     case 4:
 3599       emit_int8(0x66);
 3600     case 3:
 3601       emit_int8(0x66);
 3602     case 2:
 3603       emit_int8(0x66);
 3604     case 1:
 3605       emit_int8((unsigned char)0x90);
 3606       break;
 3607     default:
 3608       assert(i == 0, " ");
 3609   }
 3610 }
 3611 
 3612 void Assembler::notl(Register dst) {
 3613   int encode = prefix_and_encode(dst->encoding());
 3614   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 3615 }
 3616 
 3617 void Assembler::orw(Register dst, Register src) {
 3618   (void)prefix_and_encode(dst->encoding(), src->encoding());
 3619   emit_arith(0x0B, 0xC0, dst, src);
 3620 }
 3621 
 3622 void Assembler::orl(Address dst, int32_t imm32) {
 3623   InstructionMark im(this);
 3624   prefix(dst);
 3625   emit_arith_operand(0x81, rcx, dst, imm32);
 3626 }
 3627 
 3628 void Assembler::orl(Register dst, int32_t imm32) {
 3629   prefix(dst);
 3630   emit_arith(0x81, 0xC8, dst, imm32);
 3631 }
 3632 
 3633 void Assembler::orl(Register dst, Address src) {
 3634   InstructionMark im(this);
 3635   prefix(src, dst);
 3636   emit_int8(0x0B);
 3637   emit_operand(dst, src);
 3638 }
 3639 
 3640 void Assembler::orl(Register dst, Register src) {
 3641   (void) prefix_and_encode(dst->encoding(), src->encoding());
 3642   emit_arith(0x0B, 0xC0, dst, src);
 3643 }
 3644 
 3645 void Assembler::orl(Address dst, Register src) {
 3646   InstructionMark im(this);
 3647   prefix(dst, src);
 3648   emit_int8(0x09);
 3649   emit_operand(src, dst);
 3650 }
 3651 
 3652 void Assembler::orb(Address dst, int imm8) {
 3653   InstructionMark im(this);
 3654   prefix(dst);
 3655   emit_int8((unsigned char)0x80);
 3656   emit_operand(rcx, dst, 1);
 3657   emit_int8(imm8);
 3658 }
 3659 
 3660 void Assembler::orb(Address dst, Register src) {
 3661   InstructionMark im(this);
 3662   prefix(dst, src, true);
 3663   emit_int8(0x08);
 3664   emit_operand(src, dst);
 3665 }
 3666 
 3667 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 3668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3669   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3670   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3671   emit_int16(0x63, (0xC0 | encode));
 3672 }
 3673 
 3674 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3675   assert(UseAVX > 0, "some form of AVX must be enabled");
 3676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3678   emit_int16(0x63, (0xC0 | encode));
 3679 }
 3680 
 3681 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 3682   assert(VM_Version::supports_sse2(), "");
 3683   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3684   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3685   emit_int16(0x6B, (0xC0 | encode));
 3686 }
 3687 
 3688 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3689   assert(UseAVX > 0, "some form of AVX must be enabled");
 3690   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3691   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3692   emit_int16(0x6B, (0xC0 | encode));
 3693 }
 3694 
 3695 void Assembler::packuswb(XMMRegister dst, Address src) {
 3696   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3697   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 3698   InstructionMark im(this);
 3699   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3700   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 3701   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3702   emit_int8(0x67);
 3703   emit_operand(dst, src);
 3704 }
 3705 
 3706 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 3707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3708   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3709   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3710   emit_int16(0x67, (0xC0 | encode));
 3711 }
 3712 
 3713 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3714   assert(UseAVX > 0, "some form of AVX must be enabled");
 3715   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3716   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3717   emit_int16(0x67, (0xC0 | encode));
 3718 }
 3719 
 3720 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 3721   assert(VM_Version::supports_sse4_1(), "");
 3722   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3723   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3724   emit_int16(0x2B, (0xC0 | encode));
 3725 }
 3726 
 3727 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3728   assert(UseAVX > 0, "some form of AVX must be enabled");
 3729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3730   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3731   emit_int16(0x2B, (0xC0 | encode));
 3732 }
 3733 
 3734 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3735   assert(VM_Version::supports_avx2(), "");
 3736   assert(vector_len != AVX_128bit, "");
 3737   // VEX.256.66.0F3A.W1 00 /r ib
 3738   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3739   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3740   emit_int24(0x00, (0xC0 | encode), imm8);
 3741 }
 3742 
 3743 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3744   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 3745          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 3746   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3747   attributes.set_is_evex_instruction();
 3748   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3749   emit_int16(0x36, (0xC0 | encode));
 3750 }
 3751 
 3752 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3753   assert(VM_Version::supports_avx512_vbmi(), "");
 3754   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3755   attributes.set_is_evex_instruction();
 3756   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3757   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 3758 }
 3759 
 3760 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3761   assert(VM_Version::supports_avx512_vbmi(), "");
 3762   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3763   attributes.set_is_evex_instruction();
 3764   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3765   emit_int8((unsigned char)0x8D);
 3766   emit_operand(dst, src);
 3767 }
 3768 
 3769 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3770   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 3771          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 3772          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 3773   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 3774   attributes.set_is_evex_instruction();
 3775   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3776   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 3777 }
 3778 
 3779 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3780   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 3781   // VEX.NDS.256.66.0F38.W0 36 /r
 3782   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3783   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3784   emit_int16(0x36, (0xC0 | encode));
 3785 }
 3786 
 3787 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3788   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 3789   // VEX.NDS.256.66.0F38.W0 36 /r
 3790   InstructionMark im(this);
 3791   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3792   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3793   emit_int8(0x36);
 3794   emit_operand(dst, src);
 3795 }
 3796 
 3797 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 3798   assert(VM_Version::supports_avx2(), "");
 3799   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3800   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3801   emit_int24(0x46, (0xC0 | encode), imm8);
 3802 }
 3803 
 3804 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 3805   assert(VM_Version::supports_avx(), "");
 3806   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3807   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3808   emit_int24(0x06, (0xC0 | encode), imm8);
 3809 }
 3810 
 3811 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3812   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 3813   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3814   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3815   emit_int24(0x04, (0xC0 | encode), imm8);
 3816 }
 3817 
 3818 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3819   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 3820   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 3821   attributes.set_rex_vex_w_reverted();
 3822   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3823   emit_int24(0x05, (0xC0 | encode), imm8);
 3824 }
 3825 
 3826 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 3827   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 3828   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 3829   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3830   emit_int24(0x01, (0xC0 | encode), imm8);
 3831 }
 3832 
 3833 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3834   assert(VM_Version::supports_evex(), "");
 3835   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3836   attributes.set_is_evex_instruction();
 3837   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3838   emit_int16(0x76, (0xC0 | encode));
 3839 }
 3840 
 3841 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3842   assert(VM_Version::supports_avx512_vbmi(), "");
 3843   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3844   attributes.set_is_evex_instruction();
 3845   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3846   emit_int16(0x7D, (0xC0 | encode));
 3847 }
 3848 
 3849 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 3850   assert(VM_Version::supports_avx512_vbmi(), "");
 3851   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3852   attributes.set_is_evex_instruction();
 3853   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3854   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 3855 }
 3856 
 3857 void Assembler::pause() {
 3858   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 3859 }
 3860 
 3861 void Assembler::ud2() {
 3862   emit_int16(0x0F, 0x0B);
 3863 }
 3864 
 3865 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3866   assert(VM_Version::supports_sse4_2(), "");
 3867   InstructionMark im(this);
 3868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3869   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3870   emit_int8(0x61);
 3871   emit_operand(dst, src);
 3872   emit_int8(imm8);
 3873 }
 3874 
 3875 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3876   assert(VM_Version::supports_sse4_2(), "");
 3877   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3878   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3879   emit_int24(0x61, (0xC0 | encode), imm8);
 3880 }
 3881 
 3882 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 3883 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3884   assert(VM_Version::supports_sse2(), "");
 3885   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3886   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3887   emit_int16(0x74, (0xC0 | encode));
 3888 }
 3889 
 3890 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 3891   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 3892   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 3893   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3894   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3895   emit_int16(cond_encoding, (0xC0 | encode));
 3896 }
 3897 
 3898 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 3899 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3900   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 3901   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 3902   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3904   emit_int16(0x74, (0xC0 | encode));
 3905 }
 3906 
 3907 // In this context, kdst is written the mask used to process the equal components
 3908 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 3909   assert(VM_Version::supports_avx512bw(), "");
 3910   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3911   attributes.set_is_evex_instruction();
 3912   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3913   emit_int16(0x74, (0xC0 | encode));
 3914 }
 3915 
 3916 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 3917   assert(VM_Version::supports_avx512vlbw(), "");
 3918   InstructionMark im(this);
 3919   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3920   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3921   attributes.set_is_evex_instruction();
 3922   int dst_enc = kdst->encoding();
 3923   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3924   emit_int8(0x64);
 3925   emit_operand(as_Register(dst_enc), src);
 3926 }
 3927 
 3928 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 3929   assert(VM_Version::supports_avx512vlbw(), "");
 3930   InstructionMark im(this);
 3931   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3932   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3933   attributes.reset_is_clear_context();
 3934   attributes.set_embedded_opmask_register_specifier(mask);
 3935   attributes.set_is_evex_instruction();
 3936   int dst_enc = kdst->encoding();
 3937   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3938   emit_int8(0x64);
 3939   emit_operand(as_Register(dst_enc), src);
 3940 }
 3941 
 3942 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 3943   assert(VM_Version::supports_avx512vlbw(), "");
 3944   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3945   attributes.set_is_evex_instruction();
 3946   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3947   emit_int24(0x3E, (0xC0 | encode), vcc);
 3948 }
 3949 
 3950 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 3951   assert(VM_Version::supports_avx512vlbw(), "");
 3952   InstructionMark im(this);
 3953   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3954   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3955   attributes.set_is_evex_instruction();
 3956   int dst_enc = kdst->encoding();
 3957   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3958   emit_int8(0x3E);
 3959   emit_operand(as_Register(dst_enc), src);
 3960   emit_int8(vcc);
 3961 }
 3962 
 3963 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 3964   assert(VM_Version::supports_avx512bw(), "");
 3965   InstructionMark im(this);
 3966   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3967   attributes.set_is_evex_instruction();
 3968   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3969   int dst_enc = kdst->encoding();
 3970   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3971   emit_int8(0x74);
 3972   emit_operand(as_Register(dst_enc), src);
 3973 }
 3974 
 3975 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 3976   assert(VM_Version::supports_avx512vlbw(), "");
 3977   InstructionMark im(this);
 3978   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3979   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3980   attributes.reset_is_clear_context();
 3981   attributes.set_embedded_opmask_register_specifier(mask);
 3982   attributes.set_is_evex_instruction();
 3983   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3984   emit_int8(0x74);
 3985   emit_operand(as_Register(kdst->encoding()), src);
 3986 }
 3987 
 3988 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 3989 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3990   assert(VM_Version::supports_sse2(), "");
 3991   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3992   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3993   emit_int16(0x75, (0xC0 | encode));
 3994 }
 3995 
 3996 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 3997 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3998   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 3999   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4000   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4001   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4002   emit_int16(0x75, (0xC0 | encode));
 4003 }
 4004 
 4005 // In this context, kdst is written the mask used to process the equal components
 4006 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4007   assert(VM_Version::supports_avx512bw(), "");
 4008   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4009   attributes.set_is_evex_instruction();
 4010   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4011   emit_int16(0x75, (0xC0 | encode));
 4012 }
 4013 
 4014 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4015   assert(VM_Version::supports_avx512bw(), "");
 4016   InstructionMark im(this);
 4017   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4018   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4019   attributes.set_is_evex_instruction();
 4020   int dst_enc = kdst->encoding();
 4021   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4022   emit_int8(0x75);
 4023   emit_operand(as_Register(dst_enc), src);
 4024 }
 4025 
 4026 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4027 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4028   assert(VM_Version::supports_sse2(), "");
 4029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4030   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4031   emit_int16(0x76, (0xC0 | encode));
 4032 }
 4033 
 4034 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4035 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4036   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4037   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4038   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4039   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4040   emit_int16(0x76, (0xC0 | encode));
 4041 }
 4042 
 4043 // In this context, kdst is written the mask used to process the equal components
 4044 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4045   assert(VM_Version::supports_evex(), "");
 4046   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4047   attributes.set_is_evex_instruction();
 4048   attributes.reset_is_clear_context();
 4049   attributes.set_embedded_opmask_register_specifier(mask);
 4050   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4051   emit_int16(0x76, (0xC0 | encode));
 4052 }
 4053 
 4054 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4055   assert(VM_Version::supports_evex(), "");
 4056   InstructionMark im(this);
 4057   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4058   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4059   attributes.set_is_evex_instruction();
 4060   attributes.reset_is_clear_context();
 4061   attributes.set_embedded_opmask_register_specifier(mask);
 4062   int dst_enc = kdst->encoding();
 4063   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4064   emit_int8(0x76);
 4065   emit_operand(as_Register(dst_enc), src);
 4066 }
 4067 
 4068 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4069 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4070   assert(VM_Version::supports_sse4_1(), "");
 4071   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4072   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4073   emit_int16(0x29, (0xC0 | encode));
 4074 }
 4075 
 4076 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4077   assert(VM_Version::supports_avx(), "");
 4078   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4079   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4080   emit_int16(cond_encoding, (0xC0 | encode));
 4081 }
 4082 
 4083 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4084 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4085   assert(VM_Version::supports_avx(), "");
 4086   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4087   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4088   emit_int16(0x29, (0xC0 | encode));
 4089 }
 4090 
 4091 // In this context, kdst is written the mask used to process the equal components
 4092 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4093   assert(VM_Version::supports_evex(), "");
 4094   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4095   attributes.reset_is_clear_context();
 4096   attributes.set_is_evex_instruction();
 4097   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4098   emit_int16(0x29, (0xC0 | encode));
 4099 }
 4100 
 4101 // In this context, kdst is written the mask used to process the equal components
 4102 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4103   assert(VM_Version::supports_evex(), "");
 4104   InstructionMark im(this);
 4105   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4106   attributes.reset_is_clear_context();
 4107   attributes.set_is_evex_instruction();
 4108   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4109   int dst_enc = kdst->encoding();
 4110   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4111   emit_int8(0x29);
 4112   emit_operand(as_Register(dst_enc), src);
 4113 }
 4114 
 4115 void Assembler::evpmovd2m(KRegister kdst, XMMRegister src, int vector_len) {
 4116   assert(UseAVX > 2  && VM_Version::supports_avx512dq(), "");
 4117   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4118   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4119   attributes.set_is_evex_instruction();
 4120   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4121   emit_int16(0x39, (0xC0 | encode));
 4122 }
 4123 
 4124 void Assembler::evpmovq2m(KRegister kdst, XMMRegister src, int vector_len) {
 4125   assert(UseAVX > 2  && VM_Version::supports_avx512dq(), "");
 4126   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 4127   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4128   attributes.set_is_evex_instruction();
 4129   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4130   emit_int16(0x39, (0xC0 | encode));
 4131 }
 4132 
 4133 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4134   assert(VM_Version::supports_sse4_1(), "");
 4135   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4136   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4137   emit_int16(0x37, (0xC0 | encode));
 4138 }
 4139 
 4140 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4141   assert(VM_Version::supports_sse2(), "");
 4142   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4143   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4144   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4145 }
 4146 
 4147 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4148   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4149          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4150   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4151   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4152   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4153 }
 4154 
 4155 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4156   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 4157   InstructionMark im(this);
 4158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 4159   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4160   emit_int8((unsigned char)0x8C);
 4161   emit_operand(dst, src);
 4162 }
 4163 
 4164 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4165   assert(VM_Version::supports_sse4_1(), "");
 4166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4167   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4168   emit_int24(0x16, (0xC0 | encode), imm8);
 4169 }
 4170 
 4171 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4172   assert(VM_Version::supports_sse4_1(), "");
 4173   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4174   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4175   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4176   emit_int8(0x16);
 4177   emit_operand(src, dst);
 4178   emit_int8(imm8);
 4179 }
 4180 
 4181 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4182   assert(VM_Version::supports_sse4_1(), "");
 4183   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4184   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4185   emit_int24(0x16, (0xC0 | encode), imm8);
 4186 }
 4187 
 4188 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4189   assert(VM_Version::supports_sse4_1(), "");
 4190   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4191   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4192   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4193   emit_int8(0x16);
 4194   emit_operand(src, dst);
 4195   emit_int8(imm8);
 4196 }
 4197 
 4198 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4199   assert(VM_Version::supports_sse2(), "");
 4200   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4201   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4202   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4203 }
 4204 
 4205 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4206   assert(VM_Version::supports_sse4_1(), "");
 4207   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4208   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4209   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4210   emit_int8(0x15);
 4211   emit_operand(src, dst);
 4212   emit_int8(imm8);
 4213 }
 4214 
 4215 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4216   assert(VM_Version::supports_sse4_1(), "");
 4217   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4218   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4219   emit_int24(0x14, (0xC0 | encode), imm8);
 4220 }
 4221 
 4222 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4223   assert(VM_Version::supports_sse4_1(), "");
 4224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4225   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4226   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4227   emit_int8(0x14);
 4228   emit_operand(src, dst);
 4229   emit_int8(imm8);
 4230 }
 4231 
 4232 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4233   assert(VM_Version::supports_sse4_1(), "");
 4234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4235   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4236   emit_int24(0x22, (0xC0 | encode), imm8);
 4237 }
 4238 
 4239 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4240   assert(VM_Version::supports_sse4_1(), "");
 4241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4242   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4243   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4244   emit_int8(0x22);
 4245   emit_operand(dst,src);
 4246   emit_int8(imm8);
 4247 }
 4248 
 4249 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4250   assert(VM_Version::supports_avx(), "");
 4251   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4252   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4253   emit_int24(0x22, (0xC0 | encode), imm8);
 4254 }
 4255 
 4256 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4257   assert(VM_Version::supports_sse4_1(), "");
 4258   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4259   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4260   emit_int24(0x22, (0xC0 | encode), imm8);
 4261 }
 4262 
 4263 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4264   assert(VM_Version::supports_sse4_1(), "");
 4265   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4266   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4267   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4268   emit_int8(0x22);
 4269   emit_operand(dst, src);
 4270   emit_int8(imm8);
 4271 }
 4272 
 4273 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4274   assert(VM_Version::supports_avx(), "");
 4275   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4276   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4277   emit_int24(0x22, (0xC0 | encode), imm8);
 4278 }
 4279 
 4280 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4281   assert(VM_Version::supports_sse2(), "");
 4282   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4283   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4284   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4285 }
 4286 
 4287 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4288   assert(VM_Version::supports_sse2(), "");
 4289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4290   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4291   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4292   emit_int8((unsigned char)0xC4);
 4293   emit_operand(dst, src);
 4294   emit_int8(imm8);
 4295 }
 4296 
 4297 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4298   assert(VM_Version::supports_avx(), "");
 4299   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4300   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4301   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4302 }
 4303 
 4304 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4305   assert(VM_Version::supports_sse4_1(), "");
 4306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4307   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4308   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4309   emit_int8(0x20);
 4310   emit_operand(dst, src);
 4311   emit_int8(imm8);
 4312 }
 4313 
 4314 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4315   assert(VM_Version::supports_sse4_1(), "");
 4316   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4317   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4318   emit_int24(0x20, (0xC0 | encode), imm8);
 4319 }
 4320 
 4321 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4322   assert(VM_Version::supports_avx(), "");
 4323   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4324   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4325   emit_int24(0x20, (0xC0 | encode), imm8);
 4326 }
 4327 
 4328 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4329   assert(VM_Version::supports_sse4_1(), "");
 4330   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4331   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4332   emit_int24(0x21, (0xC0 | encode), imm8);
 4333 }
 4334 
 4335 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4336   assert(VM_Version::supports_avx(), "");
 4337   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4338   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4339   emit_int24(0x21, (0xC0 | encode), imm8);
 4340 }
 4341 
 4342 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4343   assert(VM_Version::supports_sse4_1(), "");
 4344   InstructionMark im(this);
 4345   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4346   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4347   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4348   emit_int8(0x30);
 4349   emit_operand(dst, src);
 4350 }
 4351 
 4352 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4353   assert(VM_Version::supports_sse4_1(), "");
 4354   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4355   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4356   emit_int16(0x30, (0xC0 | encode));
 4357 }
 4358 
 4359 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4360   assert(VM_Version::supports_sse4_1(), "");
 4361   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4362   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4363   emit_int16(0x20, (0xC0 | encode));
 4364 }
 4365 
 4366 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4367   assert(VM_Version::supports_sse4_1(), "");
 4368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4369   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4370   emit_int16(0x35, (0xC0 | encode));
 4371 }
 4372 
 4373 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4374   assert(VM_Version::supports_sse4_1(), "");
 4375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4376   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4377   emit_int16(0x21, (0xC0 | encode));
 4378 }
 4379 
 4380 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4381   assert(VM_Version::supports_sse4_1(), "");
 4382   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4383   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4384   emit_int16(0x31, (0xC0 | encode));
 4385 }
 4386 
 4387 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4388   assert(VM_Version::supports_sse4_1(), "");
 4389   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4390   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4391   emit_int16(0x22, (0xC0 | encode));
 4392 }
 4393 
 4394 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4395   assert(VM_Version::supports_sse4_1(), "");
 4396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4397   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4398   emit_int16(0x23, (0xC0 | encode));
 4399 }
 4400 
 4401 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4402   assert(VM_Version::supports_avx(), "");
 4403   InstructionMark im(this);
 4404   assert(dst != xnoreg, "sanity");
 4405   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4406   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4407   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4408   emit_int8(0x30);
 4409   emit_operand(dst, src);
 4410 }
 4411 
 4412 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4413   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4414   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4415   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4416   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4417   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4418   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4419 }
 4420 
 4421 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4422   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4423   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4424   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4425   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4426   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4427   emit_int16(0x20, (0xC0 | encode));
 4428 }
 4429 
 4430 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4431   assert(VM_Version::supports_avx512vlbw(), "");
 4432   assert(dst != xnoreg, "sanity");
 4433   InstructionMark im(this);
 4434   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4435   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4436   attributes.set_embedded_opmask_register_specifier(mask);
 4437   attributes.set_is_evex_instruction();
 4438   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4439   emit_int8(0x30);
 4440   emit_operand(dst, src);
 4441 }
 4442 
 4443 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4444   assert(VM_Version::supports_evex(), "");
 4445   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4446   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4447   attributes.set_is_evex_instruction();
 4448   attributes.set_embedded_opmask_register_specifier(mask);
 4449   if (merge) {
 4450     attributes.reset_is_clear_context();
 4451   }
 4452   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4453   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4454 }
 4455 
 4456 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4457   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4458   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4459   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4460   emit_int16(0x35, (0xC0 | encode));
 4461 }
 4462 
 4463 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4464   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4465   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4466   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4467   emit_int16(0x31, (0xC0 | encode));
 4468 }
 4469 
 4470 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4471   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4472   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4473   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4474   emit_int16(0x32, (0xC0 | encode));
 4475 }
 4476 
 4477 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4478   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4479          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4480              VM_Version::supports_evex(), "");
 4481   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4482   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4483   emit_int16(0x21, (0xC0 | encode));
 4484 }
 4485 
 4486 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4487   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4488          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4489              VM_Version::supports_evex(), "");
 4490   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4491   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4492   emit_int16(0x22, (0xC0 | encode));
 4493 }
 4494 
 4495 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4496   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4497          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4498              VM_Version::supports_evex(), "");
 4499   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4500   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4501   emit_int16(0x23, (0xC0 | encode));
 4502 }
 4503 
 4504 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 4505   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4506          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4507              VM_Version::supports_evex(), "");
 4508   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4509   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4510   emit_int16(0x24, (0xC0 | encode));
 4511 }
 4512 
 4513 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4514   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4515          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4516              VM_Version::supports_evex(), "");
 4517   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4518   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4519   emit_int16(0x25, (0xC0 | encode));
 4520 }
 4521 
 4522 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 4523   assert(VM_Version::supports_avx512vlbw(), "");
 4524   assert(src != xnoreg, "sanity");
 4525   InstructionMark im(this);
 4526   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4527   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4528   attributes.set_is_evex_instruction();
 4529   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4530   emit_int8(0x30);
 4531   emit_operand(src, dst);
 4532 }
 4533 
 4534 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 4535   assert(VM_Version::supports_avx512vlbw(), "");
 4536   assert(src != xnoreg, "sanity");
 4537   InstructionMark im(this);
 4538   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4539   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4540   attributes.reset_is_clear_context();
 4541   attributes.set_embedded_opmask_register_specifier(mask);
 4542   attributes.set_is_evex_instruction();
 4543   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4544   emit_int8(0x30);
 4545   emit_operand(src, dst);
 4546 }
 4547 
 4548 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 4549   assert(VM_Version::supports_evex(), "");
 4550   assert(src != xnoreg, "sanity");
 4551   InstructionMark im(this);
 4552   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4553   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 4554   attributes.set_is_evex_instruction();
 4555   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 4556   emit_int8(0x31);
 4557   emit_operand(src, dst);
 4558 }
 4559 
 4560 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 4561   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4562   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4563   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 4564   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4565   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4566   emit_int16(0x33, (0xC0 | encode));
 4567 }
 4568 
 4569 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 4570   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4571   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4572   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4573   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4574 }
 4575 
 4576 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4577   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4578     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4579     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 4580   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4581   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4582   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 4583 }
 4584 
 4585 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 4586 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4587        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4588        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4589   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4590   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4591   emit_int16(0x04, (0xC0 | encode));
 4592 }
 4593 
 4594 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4595   assert(VM_Version::supports_evex(), "");
 4596   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 4597   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4598   attributes.set_is_evex_instruction();
 4599   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4600   emit_int16(0x52, (0xC0 | encode));
 4601 }
 4602 
 4603 // generic
 4604 void Assembler::pop(Register dst) {
 4605   int encode = prefix_and_encode(dst->encoding());
 4606   emit_int8(0x58 | encode);
 4607 }
 4608 
 4609 void Assembler::popcntl(Register dst, Address src) {
 4610   assert(VM_Version::supports_popcnt(), "must support");
 4611   InstructionMark im(this);
 4612   emit_int8((unsigned char)0xF3);
 4613   prefix(src, dst);
 4614   emit_int16(0x0F, (unsigned char)0xB8);
 4615   emit_operand(dst, src);
 4616 }
 4617 
 4618 void Assembler::popcntl(Register dst, Register src) {
 4619   assert(VM_Version::supports_popcnt(), "must support");
 4620   emit_int8((unsigned char)0xF3);
 4621   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 4622   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 4623 }
 4624 
 4625 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
 4626   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 4627   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4628   attributes.set_is_evex_instruction();
 4629   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4630   emit_int16(0x55, (0xC0 | encode));
 4631 }
 4632 
 4633 void Assembler::popf() {
 4634   emit_int8((unsigned char)0x9D);
 4635 }
 4636 
 4637 #ifndef _LP64 // no 32bit push/pop on amd64
 4638 void Assembler::popl(Address dst) {
 4639   // NOTE: this will adjust stack by 8byte on 64bits
 4640   InstructionMark im(this);
 4641   prefix(dst);
 4642   emit_int8((unsigned char)0x8F);
 4643   emit_operand(rax, dst);
 4644 }
 4645 #endif
 4646 
 4647 void Assembler::prefetchnta(Address src) {
 4648   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4649   InstructionMark im(this);
 4650   prefix(src);
 4651   emit_int16(0x0F, 0x18);
 4652   emit_operand(rax, src); // 0, src
 4653 }
 4654 
 4655 void Assembler::prefetchr(Address src) {
 4656   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4657   InstructionMark im(this);
 4658   prefix(src);
 4659   emit_int16(0x0F, 0x0D);
 4660   emit_operand(rax, src); // 0, src
 4661 }
 4662 
 4663 void Assembler::prefetcht0(Address src) {
 4664   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4665   InstructionMark im(this);
 4666   prefix(src);
 4667   emit_int16(0x0F, 0x18);
 4668   emit_operand(rcx, src); // 1, src
 4669 }
 4670 
 4671 void Assembler::prefetcht1(Address src) {
 4672   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4673   InstructionMark im(this);
 4674   prefix(src);
 4675   emit_int16(0x0F, 0x18);
 4676   emit_operand(rdx, src); // 2, src
 4677 }
 4678 
 4679 void Assembler::prefetcht2(Address src) {
 4680   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 4681   InstructionMark im(this);
 4682   prefix(src);
 4683   emit_int16(0x0F, 0x18);
 4684   emit_operand(rbx, src); // 3, src
 4685 }
 4686 
 4687 void Assembler::prefetchw(Address src) {
 4688   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 4689   InstructionMark im(this);
 4690   prefix(src);
 4691   emit_int16(0x0F, 0x0D);
 4692   emit_operand(rcx, src); // 1, src
 4693 }
 4694 
 4695 void Assembler::prefix(Prefix p) {
 4696   emit_int8(p);
 4697 }
 4698 
 4699 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 4700   assert(VM_Version::supports_ssse3(), "");
 4701   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4702   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4703   emit_int16(0x00, (0xC0 | encode));
 4704 }
 4705 
 4706 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4707   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4708          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4709          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4710   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4711   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4712   emit_int16(0x00, (0xC0 | encode));
 4713 }
 4714 
 4715 void Assembler::pshufb(XMMRegister dst, Address src) {
 4716   assert(VM_Version::supports_ssse3(), "");
 4717   InstructionMark im(this);
 4718   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4719   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4720   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4721   emit_int8(0x00);
 4722   emit_operand(dst, src);
 4723 }
 4724 
 4725 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 4726   assert(isByte(mode), "invalid value");
 4727   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4728   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 4729   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4730   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4731   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4732 }
 4733 
 4734 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 4735   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4736          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4737          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 4738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4739   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4740   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4741   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4742 }
 4743 
 4744 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 4745   assert(isByte(mode), "invalid value");
 4746   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4747   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4748   InstructionMark im(this);
 4749   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4750   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4751   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4752   emit_int8(0x70);
 4753   emit_operand(dst, src);
 4754   emit_int8(mode & 0xFF);
 4755 }
 4756 
 4757 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 4758   assert(isByte(mode), "invalid value");
 4759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4760   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4761   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 4762   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4763 }
 4764 
 4765 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 4766   assert(isByte(mode), "invalid value");
 4767   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4769   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 4770   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 4771 }
 4772 
 4773 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 4774   assert(isByte(mode), "invalid value");
 4775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4776   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4777   InstructionMark im(this);
 4778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4779   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4780   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 4781   emit_int8(0x70);
 4782   emit_operand(dst, src);
 4783   emit_int8(mode & 0xFF);
 4784 }
 4785 
 4786 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4787   assert(VM_Version::supports_evex(), "requires EVEX support");
 4788   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 4789   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4790   attributes.set_is_evex_instruction();
 4791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4792   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 4793 }
 4794 
 4795 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
 4796   assert(isByte(imm8), "invalid value");
 4797   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4798   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4799   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4800   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4801 }
 4802 
 4803 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4804   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4805   attributes.set_rex_vex_w_reverted();
 4806   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4807   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4808 }
 4809 
 4810 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
 4811   assert(isByte(imm8), "invalid value");
 4812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4814   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4815   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4816 }
 4817 
 4818 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 4819   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4820   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4821   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 4822 }
 4823 
 4824 void Assembler::psrldq(XMMRegister dst, int shift) {
 4825   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 4826   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4827   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4828   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4829   emit_int24(0x73, (0xC0 | encode), shift);
 4830 }
 4831 
 4832 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 4833   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4834          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4835          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 4836   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4837   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4838   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 4839 }
 4840 
 4841 void Assembler::pslldq(XMMRegister dst, int shift) {
 4842   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 4843   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4844   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4845   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 4846   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4847   emit_int24(0x73, (0xC0 | encode), shift);
 4848 }
 4849 
 4850 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 4851   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4852          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4853          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 4854   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4855   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4856   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 4857 }
 4858 
 4859 void Assembler::ptest(XMMRegister dst, Address src) {
 4860   assert(VM_Version::supports_sse4_1(), "");
 4861   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4862   InstructionMark im(this);
 4863   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4864   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4865   emit_int8(0x17);
 4866   emit_operand(dst, src);
 4867 }
 4868 
 4869 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 4870   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 4871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4872   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4873   emit_int8(0x17);
 4874   emit_int8((0xC0 | encode));
 4875 }
 4876 
 4877 void Assembler::vptest(XMMRegister dst, Address src) {
 4878   assert(VM_Version::supports_avx(), "");
 4879   InstructionMark im(this);
 4880   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4881   assert(dst != xnoreg, "sanity");
 4882   // swap src<->dst for encoding
 4883   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4884   emit_int8(0x17);
 4885   emit_operand(dst, src);
 4886 }
 4887 
 4888 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 4889   assert(VM_Version::supports_avx(), "");
 4890   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4891   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4892   emit_int16(0x17, (0xC0 | encode));
 4893 }
 4894 
 4895 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 4896   assert(VM_Version::supports_avx(), "");
 4897   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4898   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4899   emit_int16(0x17, (0xC0 | encode));
 4900 }
 4901 
 4902 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4903   assert(VM_Version::supports_avx512vlbw(), "");
 4904   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4905   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4906   attributes.set_is_evex_instruction();
 4907   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4908   emit_int16((unsigned char)0x26, (0xC0 | encode));
 4909 }
 4910 
 4911 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 4912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4913   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4914   InstructionMark im(this);
 4915   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 4916   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4917   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4918   emit_int8(0x60);
 4919   emit_operand(dst, src);
 4920 }
 4921 
 4922 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 4923   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4924   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 4925   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4926   emit_int16(0x60, (0xC0 | encode));
 4927 }
 4928 
 4929 void Assembler::punpckldq(XMMRegister dst, Address src) {
 4930   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4931   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4932   InstructionMark im(this);
 4933   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4934   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4935   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4936   emit_int8(0x62);
 4937   emit_operand(dst, src);
 4938 }
 4939 
 4940 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 4941   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4942   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4943   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4944   emit_int16(0x62, (0xC0 | encode));
 4945 }
 4946 
 4947 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 4948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4949   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4950   attributes.set_rex_vex_w_reverted();
 4951   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4952   emit_int16(0x6C, (0xC0 | encode));
 4953 }
 4954 
 4955 void Assembler::push(int32_t imm32) {
 4956   // in 64bits we push 64bits onto the stack but only
 4957   // take a 32bit immediate
 4958   emit_int8(0x68);
 4959   emit_int32(imm32);
 4960 }
 4961 
 4962 void Assembler::push(Register src) {
 4963   int encode = prefix_and_encode(src->encoding());
 4964   emit_int8(0x50 | encode);
 4965 }
 4966 
 4967 void Assembler::pushf() {
 4968   emit_int8((unsigned char)0x9C);
 4969 }
 4970 
 4971 #ifndef _LP64 // no 32bit push/pop on amd64
 4972 void Assembler::pushl(Address src) {
 4973   // Note this will push 64bit on 64bit
 4974   InstructionMark im(this);
 4975   prefix(src);
 4976   emit_int8((unsigned char)0xFF);
 4977   emit_operand(rsi, src);
 4978 }
 4979 #endif
 4980 
 4981 void Assembler::rcll(Register dst, int imm8) {
 4982   assert(isShiftCount(imm8), "illegal shift count");
 4983   int encode = prefix_and_encode(dst->encoding());
 4984   if (imm8 == 1) {
 4985     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 4986   } else {
 4987     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 4988   }
 4989 }
 4990 
 4991 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 4992   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 4993   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4994   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4995   emit_int16(0x53, (0xC0 | encode));
 4996 }
 4997 
 4998 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 4999   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5000   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5001   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5002   emit_int16(0x53, (0xC0 | encode));
 5003 }
 5004 
 5005 void Assembler::rdtsc() {
 5006   emit_int16(0x0F, 0x31);
 5007 }
 5008 
 5009 // copies data from [esi] to [edi] using rcx pointer sized words
 5010 // generic
 5011 void Assembler::rep_mov() {
 5012   // REP
 5013   // MOVSQ
 5014   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5015   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5016 }
 5017 
 5018 // sets rcx bytes with rax, value at [edi]
 5019 void Assembler::rep_stosb() {
 5020   // REP
 5021   // STOSB
 5022   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5023   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5024 }
 5025 
 5026 // sets rcx pointer sized words with rax, value at [edi]
 5027 // generic
 5028 void Assembler::rep_stos() {
 5029   // REP
 5030   // LP64:STOSQ, LP32:STOSD
 5031   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5032   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5033 }
 5034 
 5035 // scans rcx pointer sized words at [edi] for occurance of rax,
 5036 // generic
 5037 void Assembler::repne_scan() { // repne_scan
 5038   // SCASQ
 5039   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5040   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5041 }
 5042 
 5043 #ifdef _LP64
 5044 // scans rcx 4 byte words at [edi] for occurance of rax,
 5045 // generic
 5046 void Assembler::repne_scanl() { // repne_scan
 5047   // SCASL
 5048   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5049 }
 5050 #endif
 5051 
 5052 void Assembler::ret(int imm16) {
 5053   if (imm16 == 0) {
 5054     emit_int8((unsigned char)0xC3);
 5055   } else {
 5056     emit_int8((unsigned char)0xC2);
 5057     emit_int16(imm16);
 5058   }
 5059 }
 5060 
 5061 void Assembler::roll(Register dst, int imm8) {
 5062   assert(isShiftCount(imm8), "illegal shift count");
 5063   int encode = prefix_and_encode(dst->encoding());
 5064   if (imm8 == 1) {
 5065     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5066   } else {
 5067     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5068   }
 5069 }
 5070 
 5071 void Assembler::roll(Register dst) {
 5072   int encode = prefix_and_encode(dst->encoding());
 5073   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5074 }
 5075 
 5076 void Assembler::rorl(Register dst, int imm8) {
 5077   assert(isShiftCount(imm8), "illegal shift count");
 5078   int encode = prefix_and_encode(dst->encoding());
 5079   if (imm8 == 1) {
 5080     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5081   } else {
 5082     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5083   }
 5084 }
 5085 
 5086 void Assembler::rorl(Register dst) {
 5087   int encode = prefix_and_encode(dst->encoding());
 5088   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5089 }
 5090 
 5091 #ifdef _LP64
 5092 void Assembler::rorq(Register dst) {
 5093   int encode = prefixq_and_encode(dst->encoding());
 5094   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5095 }
 5096 
 5097 void Assembler::rorq(Register dst, int imm8) {
 5098   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5099   int encode = prefixq_and_encode(dst->encoding());
 5100   if (imm8 == 1) {
 5101     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5102   } else {
 5103     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5104   }
 5105 }
 5106 
 5107 void Assembler::rolq(Register dst) {
 5108   int encode = prefixq_and_encode(dst->encoding());
 5109   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5110 }
 5111 
 5112 void Assembler::rolq(Register dst, int imm8) {
 5113   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5114   int encode = prefixq_and_encode(dst->encoding());
 5115   if (imm8 == 1) {
 5116     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5117   } else {
 5118     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5119   }
 5120 }
 5121 #endif
 5122 
 5123 void Assembler::sahf() {
 5124 #ifdef _LP64
 5125   // Not supported in 64bit mode
 5126   ShouldNotReachHere();
 5127 #endif
 5128   emit_int8((unsigned char)0x9E);
 5129 }
 5130 
 5131 void Assembler::sall(Address dst, int imm8) {
 5132   InstructionMark im(this);
 5133   assert(isShiftCount(imm8), "illegal shift count");
 5134   prefix(dst);
 5135   if (imm8 == 1) {
 5136     emit_int8((unsigned char)0xD1);
 5137     emit_operand(as_Register(4), dst);
 5138   }
 5139   else {
 5140     emit_int8((unsigned char)0xC1);
 5141     emit_operand(as_Register(4), dst);
 5142     emit_int8(imm8);
 5143   }
 5144 }
 5145 
 5146 void Assembler::sall(Address dst) {
 5147   InstructionMark im(this);
 5148   prefix(dst);
 5149   emit_int8((unsigned char)0xD3);
 5150   emit_operand(as_Register(4), dst);
 5151 }
 5152 
 5153 void Assembler::sall(Register dst, int imm8) {
 5154   assert(isShiftCount(imm8), "illegal shift count");
 5155   int encode = prefix_and_encode(dst->encoding());
 5156   if (imm8 == 1) {
 5157     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5158   } else {
 5159     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5160   }
 5161 }
 5162 
 5163 void Assembler::sall(Register dst) {
 5164   int encode = prefix_and_encode(dst->encoding());
 5165   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5166 }
 5167 
 5168 void Assembler::sarl(Address dst, int imm8) {
 5169   assert(isShiftCount(imm8), "illegal shift count");
 5170   InstructionMark im(this);
 5171   prefix(dst);
 5172   if (imm8 == 1) {
 5173     emit_int8((unsigned char)0xD1);
 5174     emit_operand(as_Register(7), dst);
 5175   }
 5176   else {
 5177     emit_int8((unsigned char)0xC1);
 5178     emit_operand(as_Register(7), dst);
 5179     emit_int8(imm8);
 5180   }
 5181 }
 5182 
 5183 void Assembler::sarl(Address dst) {
 5184   InstructionMark im(this);
 5185   prefix(dst);
 5186   emit_int8((unsigned char)0xD3);
 5187   emit_operand(as_Register(7), dst);
 5188 }
 5189 
 5190 void Assembler::sarl(Register dst, int imm8) {
 5191   int encode = prefix_and_encode(dst->encoding());
 5192   assert(isShiftCount(imm8), "illegal shift count");
 5193   if (imm8 == 1) {
 5194     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5195   } else {
 5196     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5197   }
 5198 }
 5199 
 5200 void Assembler::sarl(Register dst) {
 5201   int encode = prefix_and_encode(dst->encoding());
 5202   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5203 }
 5204 
 5205 void Assembler::sbbl(Address dst, int32_t imm32) {
 5206   InstructionMark im(this);
 5207   prefix(dst);
 5208   emit_arith_operand(0x81, rbx, dst, imm32);
 5209 }
 5210 
 5211 void Assembler::sbbl(Register dst, int32_t imm32) {
 5212   prefix(dst);
 5213   emit_arith(0x81, 0xD8, dst, imm32);
 5214 }
 5215 
 5216 
 5217 void Assembler::sbbl(Register dst, Address src) {
 5218   InstructionMark im(this);
 5219   prefix(src, dst);
 5220   emit_int8(0x1B);
 5221   emit_operand(dst, src);
 5222 }
 5223 
 5224 void Assembler::sbbl(Register dst, Register src) {
 5225   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5226   emit_arith(0x1B, 0xC0, dst, src);
 5227 }
 5228 
 5229 void Assembler::setb(Condition cc, Register dst) {
 5230   assert(0 <= cc && cc < 16, "illegal cc");
 5231   int encode = prefix_and_encode(dst->encoding(), true);
 5232   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5233 }
 5234 
 5235 void Assembler::sete(Register dst) {
 5236   int encode = prefix_and_encode(dst->encoding(), true);
 5237   emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
 5238 }
 5239 
 5240 void Assembler::setl(Register dst) {
 5241   int encode = prefix_and_encode(dst->encoding(), true);
 5242   emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
 5243 }
 5244 
 5245 void Assembler::setne(Register dst) {
 5246   int encode = prefix_and_encode(dst->encoding(), true);
 5247   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
 5248 }
 5249 
 5250 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5251   assert(VM_Version::supports_ssse3(), "");
 5252   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5253   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5254   emit_int24(0x0F, (0xC0 | encode), imm8);
 5255 }
 5256 
 5257 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5258   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5259          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5260          0, "");
 5261   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5262   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5263   emit_int24(0x0F, (0xC0 | encode), imm8);
 5264 }
 5265 
 5266 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5267   assert(VM_Version::supports_evex(), "");
 5268   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5269   attributes.set_is_evex_instruction();
 5270   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5271   emit_int24(0x3, (0xC0 | encode), imm8);
 5272 }
 5273 
 5274 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5275   assert(VM_Version::supports_sse4_1(), "");
 5276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5277   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5278   emit_int24(0x0E, (0xC0 | encode), imm8);
 5279 }
 5280 
 5281 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5282   assert(VM_Version::supports_sha(), "");
 5283   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5284   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5285 }
 5286 
 5287 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5288   assert(VM_Version::supports_sha(), "");
 5289   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5290   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5291 }
 5292 
 5293 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5294   assert(VM_Version::supports_sha(), "");
 5295   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5296   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5297 }
 5298 
 5299 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5300   assert(VM_Version::supports_sha(), "");
 5301   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5302   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5303 }
 5304 
 5305 // xmm0 is implicit additional source to this instruction.
 5306 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5307   assert(VM_Version::supports_sha(), "");
 5308   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5309   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5310 }
 5311 
 5312 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5313   assert(VM_Version::supports_sha(), "");
 5314   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5315   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5316 }
 5317 
 5318 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5319   assert(VM_Version::supports_sha(), "");
 5320   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5321   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5322 }
 5323 
 5324 
 5325 void Assembler::shll(Register dst, int imm8) {
 5326   assert(isShiftCount(imm8), "illegal shift count");
 5327   int encode = prefix_and_encode(dst->encoding());
 5328   if (imm8 == 1 ) {
 5329     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5330   } else {
 5331     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5332   }
 5333 }
 5334 
 5335 void Assembler::shll(Register dst) {
 5336   int encode = prefix_and_encode(dst->encoding());
 5337   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5338 }
 5339 
 5340 void Assembler::shrl(Register dst, int imm8) {
 5341   assert(isShiftCount(imm8), "illegal shift count");
 5342   int encode = prefix_and_encode(dst->encoding());
 5343   if (imm8 == 1) {
 5344     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 5345   }
 5346   else {
 5347     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 5348   }
 5349 }
 5350 
 5351 void Assembler::shrl(Register dst) {
 5352   int encode = prefix_and_encode(dst->encoding());
 5353   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 5354 }
 5355 
 5356 void Assembler::shrl(Address dst) {
 5357   InstructionMark im(this);
 5358   prefix(dst);
 5359   emit_int8((unsigned char)0xD3);
 5360   emit_operand(as_Register(5), dst);
 5361 }
 5362 
 5363 void Assembler::shrl(Address dst, int imm8) {
 5364   InstructionMark im(this);
 5365   assert(isShiftCount(imm8), "illegal shift count");
 5366   prefix(dst);
 5367   if (imm8 == 1) {
 5368     emit_int8((unsigned char)0xD1);
 5369     emit_operand(as_Register(5), dst);
 5370   }
 5371   else {
 5372     emit_int8((unsigned char)0xC1);
 5373     emit_operand(as_Register(5), dst);
 5374     emit_int8(imm8);
 5375   }
 5376 }
 5377 
 5378 
 5379 void Assembler::shldl(Register dst, Register src) {
 5380   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5381   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 5382 }
 5383 
 5384 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 5385   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5386   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 5387 }
 5388 
 5389 void Assembler::shrdl(Register dst, Register src) {
 5390   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5391   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 5392 }
 5393 
 5394 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 5395   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 5396   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 5397 }
 5398 
 5399 // copies a single word from [esi] to [edi]
 5400 void Assembler::smovl() {
 5401   emit_int8((unsigned char)0xA5);
 5402 }
 5403 
 5404 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 5405   assert(VM_Version::supports_sse4_1(), "");
 5406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5407   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5408   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 5409 }
 5410 
 5411 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 5412   assert(VM_Version::supports_sse4_1(), "");
 5413   InstructionMark im(this);
 5414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5415   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5416   emit_int8(0x0B);
 5417   emit_operand(dst, src);
 5418   emit_int8((unsigned char)rmode);
 5419 }
 5420 
 5421 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 5422   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5423   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5424   attributes.set_rex_vex_w_reverted();
 5425   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5426   emit_int16(0x51, (0xC0 | encode));
 5427 }
 5428 
 5429 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 5430   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5431   InstructionMark im(this);
 5432   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5433   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5434   attributes.set_rex_vex_w_reverted();
 5435   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5436   emit_int8(0x51);
 5437   emit_operand(dst, src);
 5438 }
 5439 
 5440 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 5441   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5442   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5443   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5444   emit_int16(0x51, (0xC0 | encode));
 5445 }
 5446 
 5447 void Assembler::std() {
 5448   emit_int8((unsigned char)0xFD);
 5449 }
 5450 
 5451 void Assembler::sqrtss(XMMRegister dst, Address src) {
 5452   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5453   InstructionMark im(this);
 5454   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5455   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5456   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5457   emit_int8(0x51);
 5458   emit_operand(dst, src);
 5459 }
 5460 
 5461 void Assembler::stmxcsr( Address dst) {
 5462   if (UseAVX > 0 ) {
 5463     assert(VM_Version::supports_avx(), "");
 5464     InstructionMark im(this);
 5465     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5466     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5467     emit_int8((unsigned char)0xAE);
 5468     emit_operand(as_Register(3), dst);
 5469   } else {
 5470     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5471     InstructionMark im(this);
 5472     prefix(dst);
 5473     emit_int16(0x0F, (unsigned char)0xAE);
 5474     emit_operand(as_Register(3), dst);
 5475   }
 5476 }
 5477 
 5478 void Assembler::subl(Address dst, int32_t imm32) {
 5479   InstructionMark im(this);
 5480   prefix(dst);
 5481   emit_arith_operand(0x81, rbp, dst, imm32);
 5482 }
 5483 
 5484 void Assembler::subl(Address dst, Register src) {
 5485   InstructionMark im(this);
 5486   prefix(dst, src);
 5487   emit_int8(0x29);
 5488   emit_operand(src, dst);
 5489 }
 5490 
 5491 void Assembler::subl(Register dst, int32_t imm32) {
 5492   prefix(dst);
 5493   emit_arith(0x81, 0xE8, dst, imm32);
 5494 }
 5495 
 5496 // Force generation of a 4 byte immediate value even if it fits into 8bit
 5497 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 5498   prefix(dst);
 5499   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 5500 }
 5501 
 5502 void Assembler::subl(Register dst, Address src) {
 5503   InstructionMark im(this);
 5504   prefix(src, dst);
 5505   emit_int8(0x2B);
 5506   emit_operand(dst, src);
 5507 }
 5508 
 5509 void Assembler::subl(Register dst, Register src) {
 5510   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5511   emit_arith(0x2B, 0xC0, dst, src);
 5512 }
 5513 
 5514 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 5515   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5516   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5517   attributes.set_rex_vex_w_reverted();
 5518   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5519   emit_int16(0x5C, (0xC0 | encode));
 5520 }
 5521 
 5522 void Assembler::subsd(XMMRegister dst, Address src) {
 5523   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5524   InstructionMark im(this);
 5525   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5526   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5527   attributes.set_rex_vex_w_reverted();
 5528   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5529   emit_int8(0x5C);
 5530   emit_operand(dst, src);
 5531 }
 5532 
 5533 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 5534   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 5536   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5537   emit_int16(0x5C, (0xC0 | encode));
 5538 }
 5539 
 5540 void Assembler::subss(XMMRegister dst, Address src) {
 5541   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5542   InstructionMark im(this);
 5543   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5544   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5545   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5546   emit_int8(0x5C);
 5547   emit_operand(dst, src);
 5548 }
 5549 
 5550 void Assembler::testb(Register dst, int imm8) {
 5551   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 5552   (void) prefix_and_encode(dst->encoding(), true);
 5553   emit_arith_b(0xF6, 0xC0, dst, imm8);
 5554 }
 5555 
 5556 void Assembler::testb(Address dst, int imm8) {
 5557   InstructionMark im(this);
 5558   prefix(dst);
 5559   emit_int8((unsigned char)0xF6);
 5560   emit_operand(rax, dst, 1);
 5561   emit_int8(imm8);
 5562 }
 5563 
 5564 void Assembler::testl(Register dst, int32_t imm32) {
 5565   // not using emit_arith because test
 5566   // doesn't support sign-extension of
 5567   // 8bit operands
 5568   int encode = dst->encoding();
 5569   encode = prefix_and_encode(encode);
 5570   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 5571   emit_int32(imm32);
 5572 }
 5573 
 5574 void Assembler::testl(Register dst, Register src) {
 5575   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5576   emit_arith(0x85, 0xC0, dst, src);
 5577 }
 5578 
 5579 void Assembler::testl(Register dst, Address src) {
 5580   InstructionMark im(this);
 5581   prefix(src, dst);
 5582   emit_int8((unsigned char)0x85);
 5583   emit_operand(dst, src);
 5584 }
 5585 
 5586 void Assembler::tzcntl(Register dst, Register src) {
 5587   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5588   emit_int8((unsigned char)0xF3);
 5589   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5590   emit_int24(0x0F,
 5591              (unsigned char)0xBC,
 5592              0xC0 | encode);
 5593 }
 5594 
 5595 void Assembler::tzcntq(Register dst, Register src) {
 5596   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 5597   emit_int8((unsigned char)0xF3);
 5598   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 5599   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 5600 }
 5601 
 5602 void Assembler::ucomisd(XMMRegister dst, Address src) {
 5603   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5604   InstructionMark im(this);
 5605   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5606   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5607   attributes.set_rex_vex_w_reverted();
 5608   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5609   emit_int8(0x2E);
 5610   emit_operand(dst, src);
 5611 }
 5612 
 5613 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 5614   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5615   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5616   attributes.set_rex_vex_w_reverted();
 5617   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5618   emit_int16(0x2E, (0xC0 | encode));
 5619 }
 5620 
 5621 void Assembler::ucomiss(XMMRegister dst, Address src) {
 5622   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5623   InstructionMark im(this);
 5624   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5625   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5626   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5627   emit_int8(0x2E);
 5628   emit_operand(dst, src);
 5629 }
 5630 
 5631 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 5632   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5633   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5634   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5635   emit_int16(0x2E, (0xC0 | encode));
 5636 }
 5637 
 5638 void Assembler::xabort(int8_t imm8) {
 5639   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 5640 }
 5641 
 5642 void Assembler::xaddb(Address dst, Register src) {
 5643   InstructionMark im(this);
 5644   prefix(dst, src, true);
 5645   emit_int16(0x0F, (unsigned char)0xC0);
 5646   emit_operand(src, dst);
 5647 }
 5648 
 5649 void Assembler::xaddw(Address dst, Register src) {
 5650   InstructionMark im(this);
 5651   emit_int8(0x66);
 5652   prefix(dst, src);
 5653   emit_int16(0x0F, (unsigned char)0xC1);
 5654   emit_operand(src, dst);
 5655 }
 5656 
 5657 void Assembler::xaddl(Address dst, Register src) {
 5658   InstructionMark im(this);
 5659   prefix(dst, src);
 5660   emit_int16(0x0F, (unsigned char)0xC1);
 5661   emit_operand(src, dst);
 5662 }
 5663 
 5664 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 5665   InstructionMark im(this);
 5666   relocate(rtype);
 5667   if (abort.is_bound()) {
 5668     address entry = target(abort);
 5669     assert(entry != NULL, "abort entry NULL");
 5670     intptr_t offset = entry - pc();
 5671     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5672     emit_int32(offset - 6); // 2 opcode + 4 address
 5673   } else {
 5674     abort.add_patch_at(code(), locator());
 5675     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 5676     emit_int32(0);
 5677   }
 5678 }
 5679 
 5680 void Assembler::xchgb(Register dst, Address src) { // xchg
 5681   InstructionMark im(this);
 5682   prefix(src, dst, true);
 5683   emit_int8((unsigned char)0x86);
 5684   emit_operand(dst, src);
 5685 }
 5686 
 5687 void Assembler::xchgw(Register dst, Address src) { // xchg
 5688   InstructionMark im(this);
 5689   emit_int8(0x66);
 5690   prefix(src, dst);
 5691   emit_int8((unsigned char)0x87);
 5692   emit_operand(dst, src);
 5693 }
 5694 
 5695 void Assembler::xchgl(Register dst, Address src) { // xchg
 5696   InstructionMark im(this);
 5697   prefix(src, dst);
 5698   emit_int8((unsigned char)0x87);
 5699   emit_operand(dst, src);
 5700 }
 5701 
 5702 void Assembler::xchgl(Register dst, Register src) {
 5703   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5704   emit_int16((unsigned char)0x87, (0xC0 | encode));
 5705 }
 5706 
 5707 void Assembler::xend() {
 5708   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 5709 }
 5710 
 5711 void Assembler::xgetbv() {
 5712   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 5713 }
 5714 
 5715 void Assembler::xorl(Address dst, int32_t imm32) {
 5716   InstructionMark im(this);
 5717   prefix(dst);
 5718   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 5719 }
 5720 
 5721 void Assembler::xorl(Register dst, int32_t imm32) {
 5722   prefix(dst);
 5723   emit_arith(0x81, 0xF0, dst, imm32);
 5724 }
 5725 
 5726 void Assembler::xorl(Register dst, Address src) {
 5727   InstructionMark im(this);
 5728   prefix(src, dst);
 5729   emit_int8(0x33);
 5730   emit_operand(dst, src);
 5731 }
 5732 
 5733 void Assembler::xorl(Register dst, Register src) {
 5734   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5735   emit_arith(0x33, 0xC0, dst, src);
 5736 }
 5737 
 5738 void Assembler::xorl(Address dst, Register src) {
 5739   InstructionMark im(this);
 5740   prefix(dst, src);
 5741   emit_int8(0x31);
 5742   emit_operand(src, dst);
 5743 }
 5744 
 5745 void Assembler::xorb(Register dst, Address src) {
 5746   InstructionMark im(this);
 5747   prefix(src, dst);
 5748   emit_int8(0x32);
 5749   emit_operand(dst, src);
 5750 }
 5751 
 5752 void Assembler::xorb(Address dst, Register src) {
 5753   InstructionMark im(this);
 5754   prefix(dst, src, true);
 5755   emit_int8(0x30);
 5756   emit_operand(src, dst);
 5757 }
 5758 
 5759 void Assembler::xorw(Register dst, Register src) {
 5760   (void)prefix_and_encode(dst->encoding(), src->encoding());
 5761   emit_arith(0x33, 0xC0, dst, src);
 5762 }
 5763 
 5764 // AVX 3-operands scalar float-point arithmetic instructions
 5765 
 5766 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 5767   assert(VM_Version::supports_avx(), "");
 5768   InstructionMark im(this);
 5769   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5770   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5771   attributes.set_rex_vex_w_reverted();
 5772   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5773   emit_int8(0x58);
 5774   emit_operand(dst, src);
 5775 }
 5776 
 5777 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5778   assert(VM_Version::supports_avx(), "");
 5779   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5780   attributes.set_rex_vex_w_reverted();
 5781   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5782   emit_int16(0x58, (0xC0 | encode));
 5783 }
 5784 
 5785 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 5786   assert(VM_Version::supports_avx(), "");
 5787   InstructionMark im(this);
 5788   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5789   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5790   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5791   emit_int8(0x58);
 5792   emit_operand(dst, src);
 5793 }
 5794 
 5795 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5796   assert(VM_Version::supports_avx(), "");
 5797   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5798   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5799   emit_int16(0x58, (0xC0 | encode));
 5800 }
 5801 
 5802 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 5803   assert(VM_Version::supports_avx(), "");
 5804   InstructionMark im(this);
 5805   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5806   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5807   attributes.set_rex_vex_w_reverted();
 5808   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5809   emit_int8(0x5E);
 5810   emit_operand(dst, src);
 5811 }
 5812 
 5813 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5814   assert(VM_Version::supports_avx(), "");
 5815   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5816   attributes.set_rex_vex_w_reverted();
 5817   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5818   emit_int16(0x5E, (0xC0 | encode));
 5819 }
 5820 
 5821 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 5822   assert(VM_Version::supports_avx(), "");
 5823   InstructionMark im(this);
 5824   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5825   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5826   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5827   emit_int8(0x5E);
 5828   emit_operand(dst, src);
 5829 }
 5830 
 5831 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5832   assert(VM_Version::supports_avx(), "");
 5833   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5834   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5835   emit_int16(0x5E, (0xC0 | encode));
 5836 }
 5837 
 5838 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 5839   assert(VM_Version::supports_fma(), "");
 5840   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5841   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5842   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 5843 }
 5844 
 5845 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 5846   assert(VM_Version::supports_fma(), "");
 5847   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5848   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5849   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 5850 }
 5851 
 5852 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 5853   assert(VM_Version::supports_avx(), "");
 5854   InstructionMark im(this);
 5855   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5856   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5857   attributes.set_rex_vex_w_reverted();
 5858   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5859   emit_int8(0x59);
 5860   emit_operand(dst, src);
 5861 }
 5862 
 5863 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5864   assert(VM_Version::supports_avx(), "");
 5865   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5866   attributes.set_rex_vex_w_reverted();
 5867   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5868   emit_int16(0x59, (0xC0 | encode));
 5869 }
 5870 
 5871 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 5872   assert(VM_Version::supports_avx(), "");
 5873   InstructionMark im(this);
 5874   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5875   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5876   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5877   emit_int8(0x59);
 5878   emit_operand(dst, src);
 5879 }
 5880 
 5881 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5882   assert(VM_Version::supports_avx(), "");
 5883   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5884   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5885   emit_int16(0x59, (0xC0 | encode));
 5886 }
 5887 
 5888 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 5889   assert(VM_Version::supports_avx(), "");
 5890   InstructionMark im(this);
 5891   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5892   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 5893   attributes.set_rex_vex_w_reverted();
 5894   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5895   emit_int8(0x5C);
 5896   emit_operand(dst, src);
 5897 }
 5898 
 5899 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5900   assert(VM_Version::supports_avx(), "");
 5901   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5902   attributes.set_rex_vex_w_reverted();
 5903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5904   emit_int16(0x5C, (0xC0 | encode));
 5905 }
 5906 
 5907 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 5908   assert(VM_Version::supports_avx(), "");
 5909   InstructionMark im(this);
 5910   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5911   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 5912   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5913   emit_int8(0x5C);
 5914   emit_operand(dst, src);
 5915 }
 5916 
 5917 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 5918   assert(VM_Version::supports_avx(), "");
 5919   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 5920   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5921   emit_int16(0x5C, (0xC0 | encode));
 5922 }
 5923 
 5924 //====================VECTOR ARITHMETIC=====================================
 5925 
 5926 // Float-point vector arithmetic
 5927 
 5928 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 5929   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5930   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5931   attributes.set_rex_vex_w_reverted();
 5932   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5933   emit_int16(0x58, (0xC0 | encode));
 5934 }
 5935 
 5936 void Assembler::addpd(XMMRegister dst, Address src) {
 5937   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5938   InstructionMark im(this);
 5939   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5940   attributes.set_rex_vex_w_reverted();
 5941   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 5942   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5943   emit_int8(0x58);
 5944   emit_operand(dst, src);
 5945 }
 5946 
 5947 
 5948 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 5949   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5950   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5951   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5952   emit_int16(0x58, (0xC0 | encode));
 5953 }
 5954 
 5955 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5956   assert(VM_Version::supports_avx(), "");
 5957   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5958   attributes.set_rex_vex_w_reverted();
 5959   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5960   emit_int16(0x58, (0xC0 | encode));
 5961 }
 5962 
 5963 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5964   assert(VM_Version::supports_avx(), "");
 5965   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5966   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5967   emit_int16(0x58, (0xC0 | encode));
 5968 }
 5969 
 5970 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 5971   assert(VM_Version::supports_avx(), "");
 5972   InstructionMark im(this);
 5973   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5974   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 5975   attributes.set_rex_vex_w_reverted();
 5976   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5977   emit_int8(0x58);
 5978   emit_operand(dst, src);
 5979 }
 5980 
 5981 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 5982   assert(VM_Version::supports_avx(), "");
 5983   InstructionMark im(this);
 5984   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5985   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5986   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5987   emit_int8(0x58);
 5988   emit_operand(dst, src);
 5989 }
 5990 
 5991 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 5992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5993   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5994   attributes.set_rex_vex_w_reverted();
 5995   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5996   emit_int16(0x5C, (0xC0 | encode));
 5997 }
 5998 
 5999 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6000   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6001   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6002   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6003   emit_int16(0x5C, (0xC0 | encode));
 6004 }
 6005 
 6006 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6007   assert(VM_Version::supports_avx(), "");
 6008   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6009   attributes.set_rex_vex_w_reverted();
 6010   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6011   emit_int16(0x5C, (0xC0 | encode));
 6012 }
 6013 
 6014 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6015   assert(VM_Version::supports_avx(), "");
 6016   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6017   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6018   emit_int16(0x5C, (0xC0 | encode));
 6019 }
 6020 
 6021 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6022   assert(VM_Version::supports_avx(), "");
 6023   InstructionMark im(this);
 6024   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6025   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6026   attributes.set_rex_vex_w_reverted();
 6027   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6028   emit_int8(0x5C);
 6029   emit_operand(dst, src);
 6030 }
 6031 
 6032 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6033   assert(VM_Version::supports_avx(), "");
 6034   InstructionMark im(this);
 6035   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6036   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6037   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6038   emit_int8(0x5C);
 6039   emit_operand(dst, src);
 6040 }
 6041 
 6042 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6043   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6044   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6045   attributes.set_rex_vex_w_reverted();
 6046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6047   emit_int16(0x59, (0xC0 | encode));
 6048 }
 6049 
 6050 void Assembler::mulpd(XMMRegister dst, Address src) {
 6051   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6052   InstructionMark im(this);
 6053   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6054   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6055   attributes.set_rex_vex_w_reverted();
 6056   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6057   emit_int8(0x59);
 6058   emit_operand(dst, src);
 6059 }
 6060 
 6061 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6062   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6063   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6064   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6065   emit_int16(0x59, (0xC0 | encode));
 6066 }
 6067 
 6068 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6069   assert(VM_Version::supports_avx(), "");
 6070   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6071   attributes.set_rex_vex_w_reverted();
 6072   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6073   emit_int16(0x59, (0xC0 | encode));
 6074 }
 6075 
 6076 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6077   assert(VM_Version::supports_avx(), "");
 6078   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6079   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6080   emit_int16(0x59, (0xC0 | encode));
 6081 }
 6082 
 6083 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6084   assert(VM_Version::supports_avx(), "");
 6085   InstructionMark im(this);
 6086   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6087   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6088   attributes.set_rex_vex_w_reverted();
 6089   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6090   emit_int8(0x59);
 6091   emit_operand(dst, src);
 6092 }
 6093 
 6094 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6095   assert(VM_Version::supports_avx(), "");
 6096   InstructionMark im(this);
 6097   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6098   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6099   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6100   emit_int8(0x59);
 6101   emit_operand(dst, src);
 6102 }
 6103 
 6104 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6105   assert(VM_Version::supports_fma(), "");
 6106   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6107   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6108   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6109 }
 6110 
 6111 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6112   assert(VM_Version::supports_fma(), "");
 6113   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6114   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6115   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6116 }
 6117 
 6118 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6119   assert(VM_Version::supports_fma(), "");
 6120   InstructionMark im(this);
 6121   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6122   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6123   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6124   emit_int8((unsigned char)0xB8);
 6125   emit_operand(dst, src2);
 6126 }
 6127 
 6128 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6129   assert(VM_Version::supports_fma(), "");
 6130   InstructionMark im(this);
 6131   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6132   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6133   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6134   emit_int8((unsigned char)0xB8);
 6135   emit_operand(dst, src2);
 6136 }
 6137 
 6138 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6140   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6141   attributes.set_rex_vex_w_reverted();
 6142   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6143   emit_int16(0x5E, (0xC0 | encode));
 6144 }
 6145 
 6146 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6147   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6148   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6149   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6150   emit_int16(0x5E, (0xC0 | encode));
 6151 }
 6152 
 6153 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6154   assert(VM_Version::supports_avx(), "");
 6155   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6156   attributes.set_rex_vex_w_reverted();
 6157   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6158   emit_int16(0x5E, (0xC0 | encode));
 6159 }
 6160 
 6161 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6162   assert(VM_Version::supports_avx(), "");
 6163   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6164   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6165   emit_int16(0x5E, (0xC0 | encode));
 6166 }
 6167 
 6168 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6169   assert(VM_Version::supports_avx(), "");
 6170   InstructionMark im(this);
 6171   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6172   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6173   attributes.set_rex_vex_w_reverted();
 6174   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6175   emit_int8(0x5E);
 6176   emit_operand(dst, src);
 6177 }
 6178 
 6179 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6180   assert(VM_Version::supports_avx(), "");
 6181   InstructionMark im(this);
 6182   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6183   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6184   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6185   emit_int8(0x5E);
 6186   emit_operand(dst, src);
 6187 }
 6188 
 6189 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6190   assert(VM_Version::supports_avx(), "");
 6191   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6192   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6193   emit_int24(0x09, (0xC0 | encode), (rmode));
 6194 }
 6195 
 6196 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6197   assert(VM_Version::supports_avx(), "");
 6198   InstructionMark im(this);
 6199   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6200   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6201   emit_int8(0x09);
 6202   emit_operand(dst, src);
 6203   emit_int8((rmode));
 6204 }
 6205 
 6206 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6207   assert(VM_Version::supports_evex(), "requires EVEX support");
 6208   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6209   attributes.set_is_evex_instruction();
 6210   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6211   emit_int24(0x09, (0xC0 | encode), (rmode));
 6212 }
 6213 
 6214 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6215   assert(VM_Version::supports_evex(), "requires EVEX support");
 6216   assert(dst != xnoreg, "sanity");
 6217   InstructionMark im(this);
 6218   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6219   attributes.set_is_evex_instruction();
 6220   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6221   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6222   emit_int8(0x09);
 6223   emit_operand(dst, src);
 6224   emit_int8((rmode));
 6225 }
 6226 
 6227 
 6228 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6229   assert(VM_Version::supports_avx(), "");
 6230   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6231   attributes.set_rex_vex_w_reverted();
 6232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6233   emit_int16(0x51, (0xC0 | encode));
 6234 }
 6235 
 6236 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6237   assert(VM_Version::supports_avx(), "");
 6238   InstructionMark im(this);
 6239   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6240   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6241   attributes.set_rex_vex_w_reverted();
 6242   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6243   emit_int8(0x51);
 6244   emit_operand(dst, src);
 6245 }
 6246 
 6247 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 6248   assert(VM_Version::supports_avx(), "");
 6249   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6250   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6251   emit_int16(0x51, (0xC0 | encode));
 6252 }
 6253 
 6254 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 6255   assert(VM_Version::supports_avx(), "");
 6256   InstructionMark im(this);
 6257   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6258   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6259   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6260   emit_int8(0x51);
 6261   emit_operand(dst, src);
 6262 }
 6263 
 6264 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 6265   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6266   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6267   attributes.set_rex_vex_w_reverted();
 6268   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6269   emit_int16(0x54, (0xC0 | encode));
 6270 }
 6271 
 6272 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 6273   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6274   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6275   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6276   emit_int16(0x54, (0xC0 | encode));
 6277 }
 6278 
 6279 void Assembler::andps(XMMRegister dst, Address src) {
 6280   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6281   InstructionMark im(this);
 6282   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6283   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6284   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6285   emit_int8(0x54);
 6286   emit_operand(dst, src);
 6287 }
 6288 
 6289 void Assembler::andpd(XMMRegister dst, Address src) {
 6290   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6291   InstructionMark im(this);
 6292   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6293   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6294   attributes.set_rex_vex_w_reverted();
 6295   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6296   emit_int8(0x54);
 6297   emit_operand(dst, src);
 6298 }
 6299 
 6300 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6301   assert(VM_Version::supports_avx(), "");
 6302   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6303   attributes.set_rex_vex_w_reverted();
 6304   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6305   emit_int16(0x54, (0xC0 | encode));
 6306 }
 6307 
 6308 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6309   assert(VM_Version::supports_avx(), "");
 6310   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6311   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6312   emit_int16(0x54, (0xC0 | encode));
 6313 }
 6314 
 6315 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6316   assert(VM_Version::supports_avx(), "");
 6317   InstructionMark im(this);
 6318   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6319   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6320   attributes.set_rex_vex_w_reverted();
 6321   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6322   emit_int8(0x54);
 6323   emit_operand(dst, src);
 6324 }
 6325 
 6326 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6327   assert(VM_Version::supports_avx(), "");
 6328   InstructionMark im(this);
 6329   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6330   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6331   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6332   emit_int8(0x54);
 6333   emit_operand(dst, src);
 6334 }
 6335 
 6336 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 6337   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6338   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6339   attributes.set_rex_vex_w_reverted();
 6340   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6341   emit_int8(0x15);
 6342   emit_int8((0xC0 | encode));
 6343 }
 6344 
 6345 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 6346   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6347   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6348   attributes.set_rex_vex_w_reverted();
 6349   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6350   emit_int16(0x14, (0xC0 | encode));
 6351 }
 6352 
 6353 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 6354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6355   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6356   attributes.set_rex_vex_w_reverted();
 6357   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6358   emit_int16(0x57, (0xC0 | encode));
 6359 }
 6360 
 6361 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 6362   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6363   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6364   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6365   emit_int16(0x57, (0xC0 | encode));
 6366 }
 6367 
 6368 void Assembler::xorpd(XMMRegister dst, Address src) {
 6369   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6370   InstructionMark im(this);
 6371   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6372   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6373   attributes.set_rex_vex_w_reverted();
 6374   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6375   emit_int8(0x57);
 6376   emit_operand(dst, src);
 6377 }
 6378 
 6379 void Assembler::xorps(XMMRegister dst, Address src) {
 6380   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6381   InstructionMark im(this);
 6382   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6383   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6384   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6385   emit_int8(0x57);
 6386   emit_operand(dst, src);
 6387 }
 6388 
 6389 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6390   assert(VM_Version::supports_avx(), "");
 6391   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6392   attributes.set_rex_vex_w_reverted();
 6393   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6394   emit_int16(0x57, (0xC0 | encode));
 6395 }
 6396 
 6397 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6398   assert(VM_Version::supports_avx(), "");
 6399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6400   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6401   emit_int16(0x57, (0xC0 | encode));
 6402 }
 6403 
 6404 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6405   assert(VM_Version::supports_avx(), "");
 6406   InstructionMark im(this);
 6407   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6408   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6409   attributes.set_rex_vex_w_reverted();
 6410   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6411   emit_int8(0x57);
 6412   emit_operand(dst, src);
 6413 }
 6414 
 6415 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6416   assert(VM_Version::supports_avx(), "");
 6417   InstructionMark im(this);
 6418   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6419   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6420   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6421   emit_int8(0x57);
 6422   emit_operand(dst, src);
 6423 }
 6424 
 6425 // Integer vector arithmetic
 6426 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6427   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6428          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6429   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6430   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6431   emit_int16(0x01, (0xC0 | encode));
 6432 }
 6433 
 6434 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6435   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 6436          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 6437   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6439   emit_int16(0x02, (0xC0 | encode));
 6440 }
 6441 
 6442 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 6443   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6444   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6445   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6446   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6447 }
 6448 
 6449 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 6450   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6451   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6452   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6453   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6454 }
 6455 
 6456 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 6457   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6460   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6461 }
 6462 
 6463 void Assembler::paddd(XMMRegister dst, Address src) {
 6464   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6465   InstructionMark im(this);
 6466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6467   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6468   emit_int8((unsigned char)0xFE);
 6469   emit_operand(dst, src);
 6470 }
 6471 
 6472 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 6473   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6474   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6475   attributes.set_rex_vex_w_reverted();
 6476   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6477   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6478 }
 6479 
 6480 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 6481   assert(VM_Version::supports_sse3(), "");
 6482   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6483   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6484   emit_int16(0x01, (0xC0 | encode));
 6485 }
 6486 
 6487 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 6488   assert(VM_Version::supports_sse3(), "");
 6489   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 6490   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6491   emit_int16(0x02, (0xC0 | encode));
 6492 }
 6493 
 6494 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6495   assert(UseAVX > 0, "requires some form of AVX");
 6496   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6497   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6498   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 6499 }
 6500 
 6501 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6502   assert(UseAVX > 0, "requires some form of AVX");
 6503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6504   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6505   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 6506 }
 6507 
 6508 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6509   assert(UseAVX > 0, "requires some form of AVX");
 6510   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6511   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6512   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 6513 }
 6514 
 6515 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6516   assert(UseAVX > 0, "requires some form of AVX");
 6517   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6518   attributes.set_rex_vex_w_reverted();
 6519   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6520   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 6521 }
 6522 
 6523 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6524   assert(UseAVX > 0, "requires some form of AVX");
 6525   InstructionMark im(this);
 6526   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6527   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6528   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6529   emit_int8((unsigned char)0xFC);
 6530   emit_operand(dst, src);
 6531 }
 6532 
 6533 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6534   assert(UseAVX > 0, "requires some form of AVX");
 6535   InstructionMark im(this);
 6536   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6537   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6538   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6539   emit_int8((unsigned char)0xFD);
 6540   emit_operand(dst, src);
 6541 }
 6542 
 6543 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6544   assert(UseAVX > 0, "requires some form of AVX");
 6545   InstructionMark im(this);
 6546   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6547   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6548   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6549   emit_int8((unsigned char)0xFE);
 6550   emit_operand(dst, src);
 6551 }
 6552 
 6553 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6554   assert(UseAVX > 0, "requires some form of AVX");
 6555   InstructionMark im(this);
 6556   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6557   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6558   attributes.set_rex_vex_w_reverted();
 6559   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6560   emit_int8((unsigned char)0xD4);
 6561   emit_operand(dst, src);
 6562 }
 6563 
 6564 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 6565   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6566   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6567   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6568   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6569 }
 6570 
 6571 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 6572   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6573   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6574   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6575   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6576 }
 6577 
 6578 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 6579   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6580   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6581   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6582 }
 6583 
 6584 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 6585   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6586   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6587   attributes.set_rex_vex_w_reverted();
 6588   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6589   emit_int8((unsigned char)0xFB);
 6590   emit_int8((0xC0 | encode));
 6591 }
 6592 
 6593 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6594   assert(UseAVX > 0, "requires some form of AVX");
 6595   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6596   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6597   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 6598 }
 6599 
 6600 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6601   assert(UseAVX > 0, "requires some form of AVX");
 6602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6603   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6604   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 6605 }
 6606 
 6607 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6608   assert(UseAVX > 0, "requires some form of AVX");
 6609   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6610   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6611   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 6612 }
 6613 
 6614 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6615   assert(UseAVX > 0, "requires some form of AVX");
 6616   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6617   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6618   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 6619 }
 6620 
 6621 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6622   assert(UseAVX > 0, "requires some form of AVX");
 6623   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6624   attributes.set_rex_vex_w_reverted();
 6625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6626   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 6627 }
 6628 
 6629 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6630   assert(UseAVX > 0, "requires some form of AVX");
 6631   InstructionMark im(this);
 6632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6633   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6634   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6635   emit_int8((unsigned char)0xF8);
 6636   emit_operand(dst, src);
 6637 }
 6638 
 6639 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6640   assert(UseAVX > 0, "requires some form of AVX");
 6641   InstructionMark im(this);
 6642   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6643   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6644   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6645   emit_int8((unsigned char)0xF9);
 6646   emit_operand(dst, src);
 6647 }
 6648 
 6649 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6650   assert(UseAVX > 0, "requires some form of AVX");
 6651   InstructionMark im(this);
 6652   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6653   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6654   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6655   emit_int8((unsigned char)0xFA);
 6656   emit_operand(dst, src);
 6657 }
 6658 
 6659 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6660   assert(UseAVX > 0, "requires some form of AVX");
 6661   InstructionMark im(this);
 6662   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6663   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6664   attributes.set_rex_vex_w_reverted();
 6665   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6666   emit_int8((unsigned char)0xFB);
 6667   emit_operand(dst, src);
 6668 }
 6669 
 6670 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 6671   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6672   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6673   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6674   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6675 }
 6676 
 6677 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 6678   assert(VM_Version::supports_sse4_1(), "");
 6679   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6680   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6681   emit_int16(0x40, (0xC0 | encode));
 6682 }
 6683 
 6684 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 6685   assert(VM_Version::supports_sse2(), "");
 6686   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6687   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6688   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6689 }
 6690 
 6691 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6692   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 6693          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 6694          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 6695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6696   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6697   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 6698 }
 6699 
 6700 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6701   assert(UseAVX > 0, "requires some form of AVX");
 6702   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6703   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6704   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 6705 }
 6706 
 6707 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6708   assert(UseAVX > 0, "requires some form of AVX");
 6709   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6710   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6711   emit_int16(0x40, (0xC0 | encode));
 6712 }
 6713 
 6714 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6715   assert(UseAVX > 2, "requires some form of EVEX");
 6716   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6717   attributes.set_is_evex_instruction();
 6718   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6719   emit_int16(0x40, (0xC0 | encode));
 6720 }
 6721 
 6722 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6723   assert(UseAVX > 0, "requires some form of AVX");
 6724   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6725   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6726   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 6727 }
 6728 
 6729 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6730   assert(UseAVX > 0, "requires some form of AVX");
 6731   InstructionMark im(this);
 6732   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6733   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 6734   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6735   emit_int8((unsigned char)0xD5);
 6736   emit_operand(dst, src);
 6737 }
 6738 
 6739 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6740   assert(UseAVX > 0, "requires some form of AVX");
 6741   InstructionMark im(this);
 6742   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6743   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6744   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6745   emit_int8(0x40);
 6746   emit_operand(dst, src);
 6747 }
 6748 
 6749 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6750   assert(UseAVX > 2, "requires some form of EVEX");
 6751   InstructionMark im(this);
 6752   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 6753   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6754   attributes.set_is_evex_instruction();
 6755   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6756   emit_int8(0x40);
 6757   emit_operand(dst, src);
 6758 }
 6759 
 6760 // Min, max
 6761 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 6762   assert(VM_Version::supports_sse4_1(), "");
 6763   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6764   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6765   emit_int16(0x38, (0xC0 | encode));
 6766 }
 6767 
 6768 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6769   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6770         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6771   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6772   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6773   emit_int16(0x38, (0xC0 | encode));
 6774 }
 6775 
 6776 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 6777   assert(VM_Version::supports_sse2(), "");
 6778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6779   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6780   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 6781 }
 6782 
 6783 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6784   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6785         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6787   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6788   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 6789 }
 6790 
 6791 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 6792   assert(VM_Version::supports_sse4_1(), "");
 6793   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6794   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6795   emit_int16(0x39, (0xC0 | encode));
 6796 }
 6797 
 6798 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6799   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6800         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 6801   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6802   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6803   emit_int16(0x39, (0xC0 | encode));
 6804 }
 6805 
 6806 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6807   assert(UseAVX > 2, "requires AVX512F");
 6808   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6809   attributes.set_is_evex_instruction();
 6810   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6811   emit_int16(0x39, (0xC0 | encode));
 6812 }
 6813 
 6814 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 6815   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6816   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6817   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6818   emit_int16(0x5D, (0xC0 | encode));
 6819 }
 6820 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6821   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6822   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6824   emit_int16(0x5D, (0xC0 | encode));
 6825 }
 6826 
 6827 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 6828   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6830   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6831   emit_int16(0x5D, (0xC0 | encode));
 6832 }
 6833 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6834   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6835   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6836   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6837   emit_int16(0x5D, (0xC0 | encode));
 6838 }
 6839 
 6840 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 6841   assert(VM_Version::supports_sse4_1(), "");
 6842   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6843   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6844   emit_int16(0x3C, (0xC0 | encode));
 6845 }
 6846 
 6847 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6848   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6849         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6850   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6851   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6852   emit_int16(0x3C, (0xC0 | encode));
 6853 }
 6854 
 6855 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 6856   assert(VM_Version::supports_sse2(), "");
 6857   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6858   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6859   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 6860 }
 6861 
 6862 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6863   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6864         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 6865   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6866   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6867   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 6868 }
 6869 
 6870 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 6871   assert(VM_Version::supports_sse4_1(), "");
 6872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6873   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6874   emit_int16(0x3D, (0xC0 | encode));
 6875 }
 6876 
 6877 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6878   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 6879         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 6880   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6881   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6882   emit_int16(0x3D, (0xC0 | encode));
 6883 }
 6884 
 6885 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6886   assert(UseAVX > 2, "requires AVX512F");
 6887   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6888   attributes.set_is_evex_instruction();
 6889   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6890   emit_int16(0x3D, (0xC0 | encode));
 6891 }
 6892 
 6893 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 6894   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6895   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6897   emit_int16(0x5F, (0xC0 | encode));
 6898 }
 6899 
 6900 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6901   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6902   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6904   emit_int16(0x5F, (0xC0 | encode));
 6905 }
 6906 
 6907 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 6908   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6909   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6910   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6911   emit_int16(0x5F, (0xC0 | encode));
 6912 }
 6913 
 6914 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6915   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 6916   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6917   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6918   emit_int16(0x5F, (0xC0 | encode));
 6919 }
 6920 
 6921 // Shift packed integers left by specified number of bits.
 6922 void Assembler::psllw(XMMRegister dst, int shift) {
 6923   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6924   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6925   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 6926   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6927   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 6928 }
 6929 
 6930 void Assembler::pslld(XMMRegister dst, int shift) {
 6931   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6932   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6933   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 6934   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6935   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 6936 }
 6937 
 6938 void Assembler::psllq(XMMRegister dst, int shift) {
 6939   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6940   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6941   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 6942   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6943   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 6944 }
 6945 
 6946 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 6947   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6949   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6950   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 6951 }
 6952 
 6953 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 6954   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6955   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6956   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6957   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 6958 }
 6959 
 6960 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 6961   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6962   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6963   attributes.set_rex_vex_w_reverted();
 6964   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6965   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 6966 }
 6967 
 6968 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 6969   assert(UseAVX > 0, "requires some form of AVX");
 6970   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6971   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 6972   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6973   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 6974 }
 6975 
 6976 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 6977   assert(UseAVX > 0, "requires some form of AVX");
 6978   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6979   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6980   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 6981   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6982   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 6983 }
 6984 
 6985 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 6986   assert(UseAVX > 0, "requires some form of AVX");
 6987   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6988   attributes.set_rex_vex_w_reverted();
 6989   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 6990   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6991   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 6992 }
 6993 
 6994 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 6995   assert(UseAVX > 0, "requires some form of AVX");
 6996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 6997   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6998   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 6999 }
 7000 
 7001 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7002   assert(UseAVX > 0, "requires some form of AVX");
 7003   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7004   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7005   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7006 }
 7007 
 7008 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7009   assert(UseAVX > 0, "requires some form of AVX");
 7010   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7011   attributes.set_rex_vex_w_reverted();
 7012   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7013   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7014 }
 7015 
 7016 // Shift packed integers logically right by specified number of bits.
 7017 void Assembler::psrlw(XMMRegister dst, int shift) {
 7018   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7019   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7020   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7021   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7022   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7023 }
 7024 
 7025 void Assembler::psrld(XMMRegister dst, int shift) {
 7026   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7027   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7028   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7029   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7030   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7031 }
 7032 
 7033 void Assembler::psrlq(XMMRegister dst, int shift) {
 7034   // Do not confuse it with psrldq SSE2 instruction which
 7035   // shifts 128 bit value in xmm register by number of bytes.
 7036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7037   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7038   attributes.set_rex_vex_w_reverted();
 7039   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7040   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7041   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7042 }
 7043 
 7044 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7045   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7046   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7047   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7048   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7049 }
 7050 
 7051 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7052   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7053   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7054   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7055   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7056 }
 7057 
 7058 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7059   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7060   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7061   attributes.set_rex_vex_w_reverted();
 7062   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7063   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7064 }
 7065 
 7066 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7067   assert(UseAVX > 0, "requires some form of AVX");
 7068   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7069   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7070   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7071   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7072 }
 7073 
 7074 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7075   assert(UseAVX > 0, "requires some form of AVX");
 7076   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7077   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7078   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7079   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7080 }
 7081 
 7082 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7083   assert(UseAVX > 0, "requires some form of AVX");
 7084   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7085   attributes.set_rex_vex_w_reverted();
 7086   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7087   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7088   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7089 }
 7090 
 7091 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7092   assert(UseAVX > 0, "requires some form of AVX");
 7093   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7094   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7095   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7096 }
 7097 
 7098 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7099   assert(UseAVX > 0, "requires some form of AVX");
 7100   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7101   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7102   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7103 }
 7104 
 7105 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7106   assert(UseAVX > 0, "requires some form of AVX");
 7107   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7108   attributes.set_rex_vex_w_reverted();
 7109   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7110   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7111 }
 7112 
 7113 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7114   assert(VM_Version::supports_avx512bw(), "");
 7115   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7116   attributes.set_is_evex_instruction();
 7117   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7118   emit_int16(0x10, (0xC0 | encode));
 7119 }
 7120 
 7121 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7122   assert(VM_Version::supports_avx512bw(), "");
 7123   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7124   attributes.set_is_evex_instruction();
 7125   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7126   emit_int16(0x12, (0xC0 | encode));
 7127 }
 7128 
 7129 // Shift packed integers arithmetically right by specified number of bits.
 7130 void Assembler::psraw(XMMRegister dst, int shift) {
 7131   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7133   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7134   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7135   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7136 }
 7137 
 7138 void Assembler::psrad(XMMRegister dst, int shift) {
 7139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7140   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7141   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7142   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7143   emit_int8(0x72);
 7144   emit_int8((0xC0 | encode));
 7145   emit_int8(shift & 0xFF);
 7146 }
 7147 
 7148 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7149   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7151   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7152   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7153 }
 7154 
 7155 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7156   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7158   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7159   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7160 }
 7161 
 7162 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7163   assert(UseAVX > 0, "requires some form of AVX");
 7164   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7165   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7166   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7167   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7168 }
 7169 
 7170 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7171   assert(UseAVX > 0, "requires some form of AVX");
 7172   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7173   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7174   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7175   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7176 }
 7177 
 7178 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7179   assert(UseAVX > 0, "requires some form of AVX");
 7180   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7181   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7182   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7183 }
 7184 
 7185 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7186   assert(UseAVX > 0, "requires some form of AVX");
 7187   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7188   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7189   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7190 }
 7191 
 7192 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7193   assert(UseAVX > 2, "requires AVX512");
 7194   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7195   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7196   attributes.set_is_evex_instruction();
 7197   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7198   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7199 }
 7200 
 7201 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7202   assert(UseAVX > 2, "requires AVX512");
 7203   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7204   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7205   attributes.set_is_evex_instruction();
 7206   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7207   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7208 }
 7209 
 7210 // logical operations packed integers
 7211 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7212   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7213   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7214   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7215   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7216 }
 7217 
 7218 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7219   assert(UseAVX > 0, "requires some form of AVX");
 7220   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7221   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7222   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7223 }
 7224 
 7225 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7226   assert(UseAVX > 0, "requires some form of AVX");
 7227   InstructionMark im(this);
 7228   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7229   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7230   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7231   emit_int8((unsigned char)0xDB);
 7232   emit_operand(dst, src);
 7233 }
 7234 
 7235 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7236   assert(VM_Version::supports_evex(), "");
 7237   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7238   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7239   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7240 }
 7241 
 7242 //Variable Shift packed integers logically left.
 7243 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7244   assert(UseAVX > 1, "requires AVX2");
 7245   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7246   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7247   emit_int16(0x47, (0xC0 | encode));
 7248 }
 7249 
 7250 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7251   assert(UseAVX > 1, "requires AVX2");
 7252   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7253   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7254   emit_int16(0x47, (0xC0 | encode));
 7255 }
 7256 
 7257 //Variable Shift packed integers logically right.
 7258 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7259   assert(UseAVX > 1, "requires AVX2");
 7260   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7261   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7262   emit_int16(0x45, (0xC0 | encode));
 7263 }
 7264 
 7265 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7266   assert(UseAVX > 1, "requires AVX2");
 7267   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7268   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7269   emit_int16(0x45, (0xC0 | encode));
 7270 }
 7271 
 7272 //Variable right Shift arithmetic packed integers .
 7273 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7274   assert(UseAVX > 1, "requires AVX2");
 7275   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7276   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7277   emit_int16(0x46, (0xC0 | encode));
 7278 }
 7279 
 7280 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7281   assert(VM_Version::supports_avx512bw(), "");
 7282   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7283   attributes.set_is_evex_instruction();
 7284   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7285   emit_int16(0x11, (0xC0 | encode));
 7286 }
 7287 
 7288 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7289   assert(UseAVX > 2, "requires AVX512");
 7290   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 7291   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7292   attributes.set_is_evex_instruction();
 7293   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7294   emit_int16(0x46, (0xC0 | encode));
 7295 }
 7296 
 7297 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7298   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7299   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7300   attributes.set_is_evex_instruction();
 7301   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7302   emit_int16(0x71, (0xC0 | encode));
 7303 }
 7304 
 7305 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7306   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 7307   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7308   attributes.set_is_evex_instruction();
 7309   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7310   emit_int16(0x73, (0xC0 | encode));
 7311 }
 7312 
 7313 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 7314   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7315   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7316   attributes.set_rex_vex_w_reverted();
 7317   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7318   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7319 }
 7320 
 7321 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7322   assert(UseAVX > 0, "requires some form of AVX");
 7323   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7324   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7325   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 7326 }
 7327 
 7328 void Assembler::por(XMMRegister dst, XMMRegister src) {
 7329   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7330   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7331   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7332   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7333 }
 7334 
 7335 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7336   assert(UseAVX > 0, "requires some form of AVX");
 7337   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7338   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7339   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7340 }
 7341 
 7342 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7343   assert(UseAVX > 0, "requires some form of AVX");
 7344   InstructionMark im(this);
 7345   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7346   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7347   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7348   emit_int8((unsigned char)0xEB);
 7349   emit_operand(dst, src);
 7350 }
 7351 
 7352 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7353   assert(VM_Version::supports_evex(), "");
 7354   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7355   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7356   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7357 }
 7358 
 7359 
 7360 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7361   assert(VM_Version::supports_evex(), "");
 7362   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7363   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7364   attributes.set_is_evex_instruction();
 7365   attributes.set_embedded_opmask_register_specifier(mask);
 7366   if (merge) {
 7367     attributes.reset_is_clear_context();
 7368   }
 7369   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7370   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 7371 }
 7372 
 7373 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 7374   assert(VM_Version::supports_evex(), "");
 7375   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 7376   InstructionMark im(this);
 7377   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7378   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 7379   attributes.set_is_evex_instruction();
 7380   attributes.set_embedded_opmask_register_specifier(mask);
 7381   if (merge) {
 7382     attributes.reset_is_clear_context();
 7383   }
 7384   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7385   emit_int8((unsigned char)0xEB);
 7386   emit_operand(dst, src);
 7387 }
 7388 
 7389 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 7390   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7391   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7392   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7393   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7394 }
 7395 
 7396 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7397   assert(UseAVX > 0, "requires some form of AVX");
 7398   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7399   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7400   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7401 }
 7402 
 7403 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7404   assert(UseAVX > 0, "requires some form of AVX");
 7405   InstructionMark im(this);
 7406   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7407   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7408   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7409   emit_int8((unsigned char)0xEF);
 7410   emit_operand(dst, src);
 7411 }
 7412 
 7413 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7414   assert(UseAVX > 2, "requires some form of EVEX");
 7415   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7416   attributes.set_rex_vex_w_reverted();
 7417   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7418   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7419 }
 7420 
 7421 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 7422   assert(VM_Version::supports_evex(), "");
 7423   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 7424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 7425   attributes.set_is_evex_instruction();
 7426   attributes.set_embedded_opmask_register_specifier(mask);
 7427   if (merge) {
 7428     attributes.reset_is_clear_context();
 7429   }
 7430   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7431   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7432 }
 7433 
 7434 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7435   assert(VM_Version::supports_evex(), "requires EVEX support");
 7436   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7437   attributes.set_is_evex_instruction();
 7438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7439   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 7440 }
 7441 
 7442 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7443   assert(VM_Version::supports_evex(), "requires EVEX support");
 7444   assert(dst != xnoreg, "sanity");
 7445   InstructionMark im(this);
 7446   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7447   attributes.set_is_evex_instruction();
 7448   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7449   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7450   emit_int8((unsigned char)0xEF);
 7451   emit_operand(dst, src);
 7452 }
 7453 
 7454 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7455   assert(VM_Version::supports_evex(), "requires EVEX support");
 7456   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7457   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7458   attributes.set_is_evex_instruction();
 7459   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7460   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7461 }
 7462 
 7463 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7464   assert(VM_Version::supports_evex(), "requires EVEX support");
 7465   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7466   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7467   attributes.set_is_evex_instruction();
 7468   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7469   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7470 }
 7471 
 7472 // Register is a class, but it would be assigned numerical value.
 7473 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
 7474 PRAGMA_DIAG_PUSH
 7475 PRAGMA_NONNULL_IGNORED
 7476 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7477   assert(VM_Version::supports_evex(), "requires EVEX support");
 7478   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7479   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7480   attributes.set_is_evex_instruction();
 7481   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7482   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7483 }
 7484 
 7485 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7486   assert(VM_Version::supports_evex(), "requires EVEX support");
 7487   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7488   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7489   attributes.set_is_evex_instruction();
 7490   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7491   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7492 }
 7493 PRAGMA_DIAG_POP
 7494 
 7495 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7496   assert(VM_Version::supports_evex(), "requires EVEX support");
 7497   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7499   attributes.set_is_evex_instruction();
 7500   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7501   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7502 }
 7503 
 7504 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7505   assert(VM_Version::supports_evex(), "requires EVEX support");
 7506   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7507   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7508   attributes.set_is_evex_instruction();
 7509   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7510   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 7511 }
 7512 
 7513 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7514   assert(VM_Version::supports_evex(), "requires EVEX support");
 7515   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7516   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7517   attributes.set_is_evex_instruction();
 7518   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7519   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7520 }
 7521 
 7522 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7523   assert(VM_Version::supports_evex(), "requires EVEX support");
 7524   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7525   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7526   attributes.set_is_evex_instruction();
 7527   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7528   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 7529 }
 7530 
 7531 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7532   assert(VM_Version::supports_evex(), "requires EVEX support");
 7533   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7534   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7535   attributes.set_is_evex_instruction();
 7536   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7537   emit_int8(0x25);
 7538   emit_int8((unsigned char)(0xC0 | encode));
 7539   emit_int8(imm8);
 7540 }
 7541 
 7542 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 7543   assert(VM_Version::supports_evex(), "requires EVEX support");
 7544   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7545   assert(dst != xnoreg, "sanity");
 7546   InstructionMark im(this);
 7547   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7548   attributes.set_is_evex_instruction();
 7549   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7550   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7551   emit_int8(0x25);
 7552   emit_operand(dst, src3);
 7553   emit_int8(imm8);
 7554 }
 7555 
 7556 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 7557   assert(VM_Version::supports_evex(), "requires EVEX support");
 7558   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 7559   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7560   attributes.set_is_evex_instruction();
 7561   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7562   emit_int8(0x25);
 7563   emit_int8((unsigned char)(0xC0 | encode));
 7564   emit_int8(imm8);
 7565 }
 7566 
 7567 // vinserti forms
 7568 
 7569 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7570   assert(VM_Version::supports_avx2(), "");
 7571   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7572   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7573   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7574   // last byte:
 7575   // 0x00 - insert into lower 128 bits
 7576   // 0x01 - insert into upper 128 bits
 7577   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 7578 }
 7579 
 7580 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7581   assert(VM_Version::supports_avx2(), "");
 7582   assert(dst != xnoreg, "sanity");
 7583   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7584   InstructionMark im(this);
 7585   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7586   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7587   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7588   emit_int8(0x38);
 7589   emit_operand(dst, src);
 7590   // 0x00 - insert into lower 128 bits
 7591   // 0x01 - insert into upper 128 bits
 7592   emit_int8(imm8 & 0x01);
 7593 }
 7594 
 7595 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7596   assert(VM_Version::supports_evex(), "");
 7597   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7598   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7599   attributes.set_is_evex_instruction();
 7600   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7601   // imm8:
 7602   // 0x00 - insert into q0 128 bits (0..127)
 7603   // 0x01 - insert into q1 128 bits (128..255)
 7604   // 0x02 - insert into q2 128 bits (256..383)
 7605   // 0x03 - insert into q3 128 bits (384..511)
 7606   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 7607 }
 7608 
 7609 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7610   assert(VM_Version::supports_avx(), "");
 7611   assert(dst != xnoreg, "sanity");
 7612   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7613   InstructionMark im(this);
 7614   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7615   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7616   attributes.set_is_evex_instruction();
 7617   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7618   emit_int8(0x18);
 7619   emit_operand(dst, src);
 7620   // 0x00 - insert into q0 128 bits (0..127)
 7621   // 0x01 - insert into q1 128 bits (128..255)
 7622   // 0x02 - insert into q2 128 bits (256..383)
 7623   // 0x03 - insert into q3 128 bits (384..511)
 7624   emit_int8(imm8 & 0x03);
 7625 }
 7626 
 7627 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7628   assert(VM_Version::supports_evex(), "");
 7629   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7630   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7631   attributes.set_is_evex_instruction();
 7632   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7633   //imm8:
 7634   // 0x00 - insert into lower 256 bits
 7635   // 0x01 - insert into upper 256 bits
 7636   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 7637 }
 7638 
 7639 
 7640 // vinsertf forms
 7641 
 7642 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7643   assert(VM_Version::supports_avx(), "");
 7644   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7645   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7646   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7647   // imm8:
 7648   // 0x00 - insert into lower 128 bits
 7649   // 0x01 - insert into upper 128 bits
 7650   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 7651 }
 7652 
 7653 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7654   assert(VM_Version::supports_avx(), "");
 7655   assert(dst != xnoreg, "sanity");
 7656   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7657   InstructionMark im(this);
 7658   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7659   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7660   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7661   emit_int8(0x18);
 7662   emit_operand(dst, src);
 7663   // 0x00 - insert into lower 128 bits
 7664   // 0x01 - insert into upper 128 bits
 7665   emit_int8(imm8 & 0x01);
 7666 }
 7667 
 7668 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7669   assert(VM_Version::supports_avx2(), "");
 7670   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7671   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7672   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7673   // imm8:
 7674   // 0x00 - insert into q0 128 bits (0..127)
 7675   // 0x01 - insert into q1 128 bits (128..255)
 7676   // 0x02 - insert into q0 128 bits (256..383)
 7677   // 0x03 - insert into q1 128 bits (384..512)
 7678   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 7679 }
 7680 
 7681 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7682   assert(VM_Version::supports_avx(), "");
 7683   assert(dst != xnoreg, "sanity");
 7684   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7685   InstructionMark im(this);
 7686   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7687   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7688   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7689   emit_int8(0x18);
 7690   emit_operand(dst, src);
 7691   // 0x00 - insert into q0 128 bits (0..127)
 7692   // 0x01 - insert into q1 128 bits (128..255)
 7693   // 0x02 - insert into q0 128 bits (256..383)
 7694   // 0x03 - insert into q1 128 bits (384..512)
 7695   emit_int8(imm8 & 0x03);
 7696 }
 7697 
 7698 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 7699   assert(VM_Version::supports_evex(), "");
 7700   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7701   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7702   attributes.set_is_evex_instruction();
 7703   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7704   // imm8:
 7705   // 0x00 - insert into lower 256 bits
 7706   // 0x01 - insert into upper 256 bits
 7707   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 7708 }
 7709 
 7710 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 7711   assert(VM_Version::supports_evex(), "");
 7712   assert(dst != xnoreg, "sanity");
 7713   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7714   InstructionMark im(this);
 7715   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7716   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 7717   attributes.set_is_evex_instruction();
 7718   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7719   emit_int8(0x1A);
 7720   emit_operand(dst, src);
 7721   // 0x00 - insert into lower 256 bits
 7722   // 0x01 - insert into upper 256 bits
 7723   emit_int8(imm8 & 0x01);
 7724 }
 7725 
 7726 
 7727 // vextracti forms
 7728 
 7729 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7730   assert(VM_Version::supports_avx2(), "");
 7731   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7732   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7733   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7734   // imm8:
 7735   // 0x00 - extract from lower 128 bits
 7736   // 0x01 - extract from upper 128 bits
 7737   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 7738 }
 7739 
 7740 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 7741   assert(VM_Version::supports_avx2(), "");
 7742   assert(src != xnoreg, "sanity");
 7743   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7744   InstructionMark im(this);
 7745   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7746   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7747   attributes.reset_is_clear_context();
 7748   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7749   emit_int8(0x39);
 7750   emit_operand(src, dst);
 7751   // 0x00 - extract from lower 128 bits
 7752   // 0x01 - extract from upper 128 bits
 7753   emit_int8(imm8 & 0x01);
 7754 }
 7755 
 7756 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7757   assert(VM_Version::supports_evex(), "");
 7758   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7759   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7760   attributes.set_is_evex_instruction();
 7761   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7762   // imm8:
 7763   // 0x00 - extract from bits 127:0
 7764   // 0x01 - extract from bits 255:128
 7765   // 0x02 - extract from bits 383:256
 7766   // 0x03 - extract from bits 511:384
 7767   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 7768 }
 7769 
 7770 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 7771   assert(VM_Version::supports_evex(), "");
 7772   assert(src != xnoreg, "sanity");
 7773   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7774   InstructionMark im(this);
 7775   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7776   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7777   attributes.reset_is_clear_context();
 7778   attributes.set_is_evex_instruction();
 7779   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7780   emit_int8(0x39);
 7781   emit_operand(src, dst);
 7782   // 0x00 - extract from bits 127:0
 7783   // 0x01 - extract from bits 255:128
 7784   // 0x02 - extract from bits 383:256
 7785   // 0x03 - extract from bits 511:384
 7786   emit_int8(imm8 & 0x03);
 7787 }
 7788 
 7789 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7790   assert(VM_Version::supports_avx512dq(), "");
 7791   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7792   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7793   attributes.set_is_evex_instruction();
 7794   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7795   // imm8:
 7796   // 0x00 - extract from bits 127:0
 7797   // 0x01 - extract from bits 255:128
 7798   // 0x02 - extract from bits 383:256
 7799   // 0x03 - extract from bits 511:384
 7800   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 7801 }
 7802 
 7803 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7804   assert(VM_Version::supports_evex(), "");
 7805   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7806   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7807   attributes.set_is_evex_instruction();
 7808   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7809   // imm8:
 7810   // 0x00 - extract from lower 256 bits
 7811   // 0x01 - extract from upper 256 bits
 7812   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 7813 }
 7814 
 7815 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 7816   assert(VM_Version::supports_evex(), "");
 7817   assert(src != xnoreg, "sanity");
 7818   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7819   InstructionMark im(this);
 7820   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7821   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 7822   attributes.reset_is_clear_context();
 7823   attributes.set_is_evex_instruction();
 7824   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7825   emit_int8(0x38);
 7826   emit_operand(src, dst);
 7827   // 0x00 - extract from lower 256 bits
 7828   // 0x01 - extract from upper 256 bits
 7829   emit_int8(imm8 & 0x01);
 7830 }
 7831 // vextractf forms
 7832 
 7833 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7834   assert(VM_Version::supports_avx(), "");
 7835   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7836   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7837   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7838   // imm8:
 7839   // 0x00 - extract from lower 128 bits
 7840   // 0x01 - extract from upper 128 bits
 7841   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 7842 }
 7843 
 7844 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 7845   assert(VM_Version::supports_avx(), "");
 7846   assert(src != xnoreg, "sanity");
 7847   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7848   InstructionMark im(this);
 7849   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7850   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7851   attributes.reset_is_clear_context();
 7852   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7853   emit_int8(0x19);
 7854   emit_operand(src, dst);
 7855   // 0x00 - extract from lower 128 bits
 7856   // 0x01 - extract from upper 128 bits
 7857   emit_int8(imm8 & 0x01);
 7858 }
 7859 
 7860 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7861   assert(VM_Version::supports_evex(), "");
 7862   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7863   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7864   attributes.set_is_evex_instruction();
 7865   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7866   // imm8:
 7867   // 0x00 - extract from bits 127:0
 7868   // 0x01 - extract from bits 255:128
 7869   // 0x02 - extract from bits 383:256
 7870   // 0x03 - extract from bits 511:384
 7871   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 7872 }
 7873 
 7874 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 7875   assert(VM_Version::supports_evex(), "");
 7876   assert(src != xnoreg, "sanity");
 7877   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7878   InstructionMark im(this);
 7879   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7880   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 7881   attributes.reset_is_clear_context();
 7882   attributes.set_is_evex_instruction();
 7883   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7884   emit_int8(0x19);
 7885   emit_operand(src, dst);
 7886   // 0x00 - extract from bits 127:0
 7887   // 0x01 - extract from bits 255:128
 7888   // 0x02 - extract from bits 383:256
 7889   // 0x03 - extract from bits 511:384
 7890   emit_int8(imm8 & 0x03);
 7891 }
 7892 
 7893 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7894   assert(VM_Version::supports_avx512dq(), "");
 7895   assert(imm8 <= 0x03, "imm8: %u", imm8);
 7896   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7897   attributes.set_is_evex_instruction();
 7898   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7899   // imm8:
 7900   // 0x00 - extract from bits 127:0
 7901   // 0x01 - extract from bits 255:128
 7902   // 0x02 - extract from bits 383:256
 7903   // 0x03 - extract from bits 511:384
 7904   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 7905 }
 7906 
 7907 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 7908   assert(VM_Version::supports_evex(), "");
 7909   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7910   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7911   attributes.set_is_evex_instruction();
 7912   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7913   // imm8:
 7914   // 0x00 - extract from lower 256 bits
 7915   // 0x01 - extract from upper 256 bits
 7916   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 7917 }
 7918 
 7919 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 7920   assert(VM_Version::supports_evex(), "");
 7921   assert(src != xnoreg, "sanity");
 7922   assert(imm8 <= 0x01, "imm8: %u", imm8);
 7923   InstructionMark im(this);
 7924   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7925   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 7926   attributes.reset_is_clear_context();
 7927   attributes.set_is_evex_instruction();
 7928   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 7929   emit_int8(0x1B);
 7930   emit_operand(src, dst);
 7931   // 0x00 - extract from lower 256 bits
 7932   // 0x01 - extract from upper 256 bits
 7933   emit_int8(imm8 & 0x01);
 7934 }
 7935 
 7936 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 7937 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 7938   assert(VM_Version::supports_avx2(), "");
 7939   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7940   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7941   emit_int16(0x78, (0xC0 | encode));
 7942 }
 7943 
 7944 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 7945   assert(VM_Version::supports_avx2(), "");
 7946   assert(dst != xnoreg, "sanity");
 7947   InstructionMark im(this);
 7948   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7949   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 7950   // swap src<->dst for encoding
 7951   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7952   emit_int8(0x78);
 7953   emit_operand(dst, src);
 7954 }
 7955 
 7956 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 7957 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 7958   assert(VM_Version::supports_avx2(), "");
 7959   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7960   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7961   emit_int16(0x79, (0xC0 | encode));
 7962 }
 7963 
 7964 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 7965   assert(VM_Version::supports_avx2(), "");
 7966   assert(dst != xnoreg, "sanity");
 7967   InstructionMark im(this);
 7968   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7969   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 7970   // swap src<->dst for encoding
 7971   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7972   emit_int8(0x79);
 7973   emit_operand(dst, src);
 7974 }
 7975 
 7976 // xmm/mem sourced byte/word/dword/qword replicate
 7977 
 7978 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 7979 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
 7980   assert(UseAVX >= 2, "");
 7981   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7982   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7983   emit_int16(0x58, (0xC0 | encode));
 7984 }
 7985 
 7986 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
 7987   assert(VM_Version::supports_avx2(), "");
 7988   assert(dst != xnoreg, "sanity");
 7989   InstructionMark im(this);
 7990   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7991   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 7992   // swap src<->dst for encoding
 7993   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7994   emit_int8(0x58);
 7995   emit_operand(dst, src);
 7996 }
 7997 
 7998 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 7999 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
 8000   assert(VM_Version::supports_avx2(), "");
 8001   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8002   attributes.set_rex_vex_w_reverted();
 8003   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8004   emit_int16(0x59, (0xC0 | encode));
 8005 }
 8006 
 8007 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
 8008   assert(VM_Version::supports_avx2(), "");
 8009   assert(dst != xnoreg, "sanity");
 8010   InstructionMark im(this);
 8011   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8012   attributes.set_rex_vex_w_reverted();
 8013   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 8014   // swap src<->dst for encoding
 8015   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8016   emit_int8(0x59);
 8017   emit_operand(dst, src);
 8018 }
 8019 
 8020 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
 8021   assert(vector_len != Assembler::AVX_128bit, "");
 8022   assert(VM_Version::supports_avx512dq(), "");
 8023   assert(dst != xnoreg, "sanity");
 8024   InstructionMark im(this);
 8025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8026   attributes.set_rex_vex_w_reverted();
 8027   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 8028   // swap src<->dst for encoding
 8029   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8030   emit_int8(0x5A);
 8031   emit_operand(dst, src);
 8032 }
 8033 
 8034 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
 8035   assert(vector_len != Assembler::AVX_128bit, "");
 8036   assert(VM_Version::supports_avx512dq(), "");
 8037   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8038   attributes.set_rex_vex_w_reverted();
 8039   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8040   emit_int16(0x5A, (0xC0 | encode));
 8041 }
 8042 
 8043 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
 8044   assert(vector_len != Assembler::AVX_128bit, "");
 8045   assert(VM_Version::supports_avx512dq(), "");
 8046   assert(dst != xnoreg, "sanity");
 8047   InstructionMark im(this);
 8048   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8049   attributes.set_rex_vex_w_reverted();
 8050   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
 8051   // swap src<->dst for encoding
 8052   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8053   emit_int8(0x5A);
 8054   emit_operand(dst, src);
 8055 }
 8056 
 8057 // scalar single/double precision replicate
 8058 
 8059 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
 8060 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
 8061   assert(VM_Version::supports_avx2(), "");
 8062   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8063   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8064   emit_int16(0x18, (0xC0 | encode));
 8065 }
 8066 
 8067 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
 8068   assert(VM_Version::supports_avx(), "");
 8069   assert(dst != xnoreg, "sanity");
 8070   InstructionMark im(this);
 8071   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8072   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8073   // swap src<->dst for encoding
 8074   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8075   emit_int8(0x18);
 8076   emit_operand(dst, src);
 8077 }
 8078 
 8079 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
 8080 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
 8081   assert(VM_Version::supports_avx2(), "");
 8082   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 8083   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8084   attributes.set_rex_vex_w_reverted();
 8085   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8086   emit_int16(0x19, (0xC0 | encode));
 8087 }
 8088 
 8089 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
 8090   assert(VM_Version::supports_avx(), "");
 8091   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
 8092   assert(dst != xnoreg, "sanity");
 8093   InstructionMark im(this);
 8094   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8095   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 8096   attributes.set_rex_vex_w_reverted();
 8097   // swap src<->dst for encoding
 8098   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8099   emit_int8(0x19);
 8100   emit_operand(dst, src);
 8101 }
 8102 
 8103 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
 8104   assert(VM_Version::supports_avx(), "");
 8105   assert(vector_len == AVX_256bit, "");
 8106   assert(dst != xnoreg, "sanity");
 8107   InstructionMark im(this);
 8108   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8109   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8110   // swap src<->dst for encoding
 8111   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8112   emit_int8(0x1A);
 8113   emit_operand(dst, src);
 8114 }
 8115 
 8116 // gpr source broadcast forms
 8117 
 8118 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8119 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
 8120   assert(VM_Version::supports_avx512bw(), "");
 8121   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8122   attributes.set_is_evex_instruction();
 8123   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8124   emit_int16(0x7A, (0xC0 | encode));
 8125 }
 8126 
 8127 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8128 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
 8129   assert(VM_Version::supports_avx512bw(), "");
 8130   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8131   attributes.set_is_evex_instruction();
 8132   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8133   emit_int16(0x7B, (0xC0 | encode));
 8134 }
 8135 
 8136 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
 8137 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
 8138   assert(VM_Version::supports_evex(), "");
 8139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8140   attributes.set_is_evex_instruction();
 8141   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8142   emit_int16(0x7C, (0xC0 | encode));
 8143 }
 8144 
 8145 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
 8146 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
 8147   assert(VM_Version::supports_evex(), "");
 8148   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8149   attributes.set_is_evex_instruction();
 8150   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8151   emit_int16(0x7C, (0xC0 | encode));
 8152 }
 8153 
 8154 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8155   assert(VM_Version::supports_avx2(), "");
 8156   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8157   assert(dst != xnoreg, "sanity");
 8158   assert(src.isxmmindex(),"expected to be xmm index");
 8159   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8160   InstructionMark im(this);
 8161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8162   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8163   emit_int8((unsigned char)0x90);
 8164   emit_operand(dst, src);
 8165 }
 8166 
 8167 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8168   assert(VM_Version::supports_avx2(), "");
 8169   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8170   assert(dst != xnoreg, "sanity");
 8171   assert(src.isxmmindex(),"expected to be xmm index");
 8172   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8173   InstructionMark im(this);
 8174   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8175   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8176   emit_int8((unsigned char)0x90);
 8177   emit_operand(dst, src);
 8178 }
 8179 
 8180 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8181   assert(VM_Version::supports_avx2(), "");
 8182   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8183   assert(dst != xnoreg, "sanity");
 8184   assert(src.isxmmindex(),"expected to be xmm index");
 8185   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8186   InstructionMark im(this);
 8187   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8188   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8189   emit_int8((unsigned char)0x92);
 8190   emit_operand(dst, src);
 8191 }
 8192 
 8193 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 8194   assert(VM_Version::supports_avx2(), "");
 8195   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
 8196   assert(dst != xnoreg, "sanity");
 8197   assert(src.isxmmindex(),"expected to be xmm index");
 8198   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8199   InstructionMark im(this);
 8200   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
 8201   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8202   emit_int8((unsigned char)0x92);
 8203   emit_operand(dst, src);
 8204 }
 8205 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8206   assert(VM_Version::supports_evex(), "");
 8207   assert(dst != xnoreg, "sanity");
 8208   assert(src.isxmmindex(),"expected to be xmm index");
 8209   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8210   assert(mask != k0, "instruction will #UD if mask is in k0");
 8211   InstructionMark im(this);
 8212   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8213   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8214   attributes.reset_is_clear_context();
 8215   attributes.set_embedded_opmask_register_specifier(mask);
 8216   attributes.set_is_evex_instruction();
 8217   // swap src<->dst for encoding
 8218   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8219   emit_int8((unsigned char)0x90);
 8220   emit_operand(dst, src);
 8221 }
 8222 
 8223 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8224   assert(VM_Version::supports_evex(), "");
 8225   assert(dst != xnoreg, "sanity");
 8226   assert(src.isxmmindex(),"expected to be xmm index");
 8227   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8228   assert(mask != k0, "instruction will #UD if mask is in k0");
 8229   InstructionMark im(this);
 8230   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8231   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8232   attributes.reset_is_clear_context();
 8233   attributes.set_embedded_opmask_register_specifier(mask);
 8234   attributes.set_is_evex_instruction();
 8235   // swap src<->dst for encoding
 8236   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8237   emit_int8((unsigned char)0x90);
 8238   emit_operand(dst, src);
 8239 }
 8240 
 8241 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8242   assert(VM_Version::supports_evex(), "");
 8243   assert(dst != xnoreg, "sanity");
 8244   assert(src.isxmmindex(),"expected to be xmm index");
 8245   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8246   assert(mask != k0, "instruction will #UD if mask is in k0");
 8247   InstructionMark im(this);
 8248   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8249   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8250   attributes.reset_is_clear_context();
 8251   attributes.set_embedded_opmask_register_specifier(mask);
 8252   attributes.set_is_evex_instruction();
 8253   // swap src<->dst for encoding
 8254   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8255   emit_int8((unsigned char)0x92);
 8256   emit_operand(dst, src);
 8257 }
 8258 
 8259 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 8260   assert(VM_Version::supports_evex(), "");
 8261   assert(dst != xnoreg, "sanity");
 8262   assert(src.isxmmindex(),"expected to be xmm index");
 8263   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
 8264   assert(mask != k0, "instruction will #UD if mask is in k0");
 8265   InstructionMark im(this);
 8266   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8267   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8268   attributes.reset_is_clear_context();
 8269   attributes.set_embedded_opmask_register_specifier(mask);
 8270   attributes.set_is_evex_instruction();
 8271   // swap src<->dst for encoding
 8272   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8273   emit_int8((unsigned char)0x92);
 8274   emit_operand(dst, src);
 8275 }
 8276 
 8277 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8278   assert(VM_Version::supports_evex(), "");
 8279   assert(mask != k0, "instruction will #UD if mask is in k0");
 8280   InstructionMark im(this);
 8281   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8282   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8283   attributes.reset_is_clear_context();
 8284   attributes.set_embedded_opmask_register_specifier(mask);
 8285   attributes.set_is_evex_instruction();
 8286   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8287   emit_int8((unsigned char)0xA0);
 8288   emit_operand(src, dst);
 8289 }
 8290 
 8291 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8292   assert(VM_Version::supports_evex(), "");
 8293   assert(mask != k0, "instruction will #UD if mask is in k0");
 8294   InstructionMark im(this);
 8295   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8296   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8297   attributes.reset_is_clear_context();
 8298   attributes.set_embedded_opmask_register_specifier(mask);
 8299   attributes.set_is_evex_instruction();
 8300   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8301   emit_int8((unsigned char)0xA0);
 8302   emit_operand(src, dst);
 8303 }
 8304 
 8305 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8306   assert(VM_Version::supports_evex(), "");
 8307   assert(mask != k0, "instruction will #UD if mask is in k0");
 8308   InstructionMark im(this);
 8309   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8310   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8311   attributes.reset_is_clear_context();
 8312   attributes.set_embedded_opmask_register_specifier(mask);
 8313   attributes.set_is_evex_instruction();
 8314   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8315   emit_int8((unsigned char)0xA2);
 8316   emit_operand(src, dst);
 8317 }
 8318 
 8319 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 8320   assert(VM_Version::supports_evex(), "");
 8321   assert(mask != k0, "instruction will #UD if mask is in k0");
 8322   InstructionMark im(this);
 8323   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8324   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 8325   attributes.reset_is_clear_context();
 8326   attributes.set_embedded_opmask_register_specifier(mask);
 8327   attributes.set_is_evex_instruction();
 8328   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8329   emit_int8((unsigned char)0xA2);
 8330   emit_operand(src, dst);
 8331 }
 8332 // Carry-Less Multiplication Quadword
 8333 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
 8334   assert(VM_Version::supports_clmul(), "");
 8335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8337   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8338 }
 8339 
 8340 // Carry-Less Multiplication Quadword
 8341 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
 8342   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
 8343   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 8344   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8345   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8346 }
 8347 
 8348 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
 8349   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
 8350   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8351   attributes.set_is_evex_instruction();
 8352   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8353   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
 8354 }
 8355 
 8356 void Assembler::vzeroupper_uncached() {
 8357   if (VM_Version::supports_vzeroupper()) {
 8358     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 8359     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 8360     emit_int8(0x77);
 8361   }
 8362 }
 8363 
 8364 void Assembler::fld_x(Address adr) {
 8365   InstructionMark im(this);
 8366   emit_int8((unsigned char)0xDB);
 8367   emit_operand32(rbp, adr);
 8368 }
 8369 
 8370 void Assembler::fstp_x(Address adr) {
 8371   InstructionMark im(this);
 8372   emit_int8((unsigned char)0xDB);
 8373   emit_operand32(rdi, adr);
 8374 }
 8375 
 8376 void Assembler::emit_operand32(Register reg, Address adr) {
 8377   assert(reg->encoding() < 8, "no extended registers");
 8378   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 8379   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 8380                adr._rspec);
 8381 }
 8382 
 8383 #ifndef _LP64
 8384 // 32bit only pieces of the assembler
 8385 
 8386 void Assembler::emms() {
 8387   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
 8388   emit_int16(0x0F, 0x77);
 8389 }
 8390 
 8391 void Assembler::vzeroupper() {
 8392   vzeroupper_uncached();
 8393 }
 8394 
 8395 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
 8396   // NO PREFIX AS NEVER 64BIT
 8397   InstructionMark im(this);
 8398   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
 8399   emit_data(imm32, rspec, 0);
 8400 }
 8401 
 8402 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
 8403   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
 8404   InstructionMark im(this);
 8405   emit_int8((unsigned char)0x81);
 8406   emit_operand(rdi, src1);
 8407   emit_data(imm32, rspec, 0);
 8408 }
 8409 
 8410 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
 8411 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
 8412 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
 8413 void Assembler::cmpxchg8(Address adr) {
 8414   InstructionMark im(this);
 8415   emit_int16(0x0F, (unsigned char)0xC7);
 8416   emit_operand(rcx, adr);
 8417 }
 8418 
 8419 void Assembler::decl(Register dst) {
 8420   // Don't use it directly. Use MacroAssembler::decrementl() instead.
 8421  emit_int8(0x48 | dst->encoding());
 8422 }
 8423 
 8424 // 64bit doesn't use the x87
 8425 
 8426 void Assembler::emit_farith(int b1, int b2, int i) {
 8427   assert(isByte(b1) && isByte(b2), "wrong opcode");
 8428   assert(0 <= i &&  i < 8, "illegal stack offset");
 8429   emit_int16(b1, b2 + i);
 8430 }
 8431 
 8432 void Assembler::fabs() {
 8433   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
 8434 }
 8435 
 8436 void Assembler::fadd(int i) {
 8437   emit_farith(0xD8, 0xC0, i);
 8438 }
 8439 
 8440 void Assembler::fadd_d(Address src) {
 8441   InstructionMark im(this);
 8442   emit_int8((unsigned char)0xDC);
 8443   emit_operand32(rax, src);
 8444 }
 8445 
 8446 void Assembler::fadd_s(Address src) {
 8447   InstructionMark im(this);
 8448   emit_int8((unsigned char)0xD8);
 8449   emit_operand32(rax, src);
 8450 }
 8451 
 8452 void Assembler::fadda(int i) {
 8453   emit_farith(0xDC, 0xC0, i);
 8454 }
 8455 
 8456 void Assembler::faddp(int i) {
 8457   emit_farith(0xDE, 0xC0, i);
 8458 }
 8459 
 8460 void Assembler::fchs() {
 8461   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
 8462 }
 8463 
 8464 void Assembler::fcom(int i) {
 8465   emit_farith(0xD8, 0xD0, i);
 8466 }
 8467 
 8468 void Assembler::fcomp(int i) {
 8469   emit_farith(0xD8, 0xD8, i);
 8470 }
 8471 
 8472 void Assembler::fcomp_d(Address src) {
 8473   InstructionMark im(this);
 8474   emit_int8((unsigned char)0xDC);
 8475   emit_operand32(rbx, src);
 8476 }
 8477 
 8478 void Assembler::fcomp_s(Address src) {
 8479   InstructionMark im(this);
 8480   emit_int8((unsigned char)0xD8);
 8481   emit_operand32(rbx, src);
 8482 }
 8483 
 8484 void Assembler::fcompp() {
 8485   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
 8486 }
 8487 
 8488 void Assembler::fcos() {
 8489   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
 8490 }
 8491 
 8492 void Assembler::fdecstp() {
 8493   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
 8494 }
 8495 
 8496 void Assembler::fdiv(int i) {
 8497   emit_farith(0xD8, 0xF0, i);
 8498 }
 8499 
 8500 void Assembler::fdiv_d(Address src) {
 8501   InstructionMark im(this);
 8502   emit_int8((unsigned char)0xDC);
 8503   emit_operand32(rsi, src);
 8504 }
 8505 
 8506 void Assembler::fdiv_s(Address src) {
 8507   InstructionMark im(this);
 8508   emit_int8((unsigned char)0xD8);
 8509   emit_operand32(rsi, src);
 8510 }
 8511 
 8512 void Assembler::fdiva(int i) {
 8513   emit_farith(0xDC, 0xF8, i);
 8514 }
 8515 
 8516 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
 8517 //       is erroneous for some of the floating-point instructions below.
 8518 
 8519 void Assembler::fdivp(int i) {
 8520   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
 8521 }
 8522 
 8523 void Assembler::fdivr(int i) {
 8524   emit_farith(0xD8, 0xF8, i);
 8525 }
 8526 
 8527 void Assembler::fdivr_d(Address src) {
 8528   InstructionMark im(this);
 8529   emit_int8((unsigned char)0xDC);
 8530   emit_operand32(rdi, src);
 8531 }
 8532 
 8533 void Assembler::fdivr_s(Address src) {
 8534   InstructionMark im(this);
 8535   emit_int8((unsigned char)0xD8);
 8536   emit_operand32(rdi, src);
 8537 }
 8538 
 8539 void Assembler::fdivra(int i) {
 8540   emit_farith(0xDC, 0xF0, i);
 8541 }
 8542 
 8543 void Assembler::fdivrp(int i) {
 8544   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
 8545 }
 8546 
 8547 void Assembler::ffree(int i) {
 8548   emit_farith(0xDD, 0xC0, i);
 8549 }
 8550 
 8551 void Assembler::fild_d(Address adr) {
 8552   InstructionMark im(this);
 8553   emit_int8((unsigned char)0xDF);
 8554   emit_operand32(rbp, adr);
 8555 }
 8556 
 8557 void Assembler::fild_s(Address adr) {
 8558   InstructionMark im(this);
 8559   emit_int8((unsigned char)0xDB);
 8560   emit_operand32(rax, adr);
 8561 }
 8562 
 8563 void Assembler::fincstp() {
 8564   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
 8565 }
 8566 
 8567 void Assembler::finit() {
 8568   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
 8569 }
 8570 
 8571 void Assembler::fist_s(Address adr) {
 8572   InstructionMark im(this);
 8573   emit_int8((unsigned char)0xDB);
 8574   emit_operand32(rdx, adr);
 8575 }
 8576 
 8577 void Assembler::fistp_d(Address adr) {
 8578   InstructionMark im(this);
 8579   emit_int8((unsigned char)0xDF);
 8580   emit_operand32(rdi, adr);
 8581 }
 8582 
 8583 void Assembler::fistp_s(Address adr) {
 8584   InstructionMark im(this);
 8585   emit_int8((unsigned char)0xDB);
 8586   emit_operand32(rbx, adr);
 8587 }
 8588 
 8589 void Assembler::fld1() {
 8590   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
 8591 }
 8592 
 8593 void Assembler::fld_d(Address adr) {
 8594   InstructionMark im(this);
 8595   emit_int8((unsigned char)0xDD);
 8596   emit_operand32(rax, adr);
 8597 }
 8598 
 8599 void Assembler::fld_s(Address adr) {
 8600   InstructionMark im(this);
 8601   emit_int8((unsigned char)0xD9);
 8602   emit_operand32(rax, adr);
 8603 }
 8604 
 8605 
 8606 void Assembler::fld_s(int index) {
 8607   emit_farith(0xD9, 0xC0, index);
 8608 }
 8609 
 8610 void Assembler::fldcw(Address src) {
 8611   InstructionMark im(this);
 8612   emit_int8((unsigned char)0xD9);
 8613   emit_operand32(rbp, src);
 8614 }
 8615 
 8616 void Assembler::fldenv(Address src) {
 8617   InstructionMark im(this);
 8618   emit_int8((unsigned char)0xD9);
 8619   emit_operand32(rsp, src);
 8620 }
 8621 
 8622 void Assembler::fldlg2() {
 8623   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
 8624 }
 8625 
 8626 void Assembler::fldln2() {
 8627   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
 8628 }
 8629 
 8630 void Assembler::fldz() {
 8631   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
 8632 }
 8633 
 8634 void Assembler::flog() {
 8635   fldln2();
 8636   fxch();
 8637   fyl2x();
 8638 }
 8639 
 8640 void Assembler::flog10() {
 8641   fldlg2();
 8642   fxch();
 8643   fyl2x();
 8644 }
 8645 
 8646 void Assembler::fmul(int i) {
 8647   emit_farith(0xD8, 0xC8, i);
 8648 }
 8649 
 8650 void Assembler::fmul_d(Address src) {
 8651   InstructionMark im(this);
 8652   emit_int8((unsigned char)0xDC);
 8653   emit_operand32(rcx, src);
 8654 }
 8655 
 8656 void Assembler::fmul_s(Address src) {
 8657   InstructionMark im(this);
 8658   emit_int8((unsigned char)0xD8);
 8659   emit_operand32(rcx, src);
 8660 }
 8661 
 8662 void Assembler::fmula(int i) {
 8663   emit_farith(0xDC, 0xC8, i);
 8664 }
 8665 
 8666 void Assembler::fmulp(int i) {
 8667   emit_farith(0xDE, 0xC8, i);
 8668 }
 8669 
 8670 void Assembler::fnsave(Address dst) {
 8671   InstructionMark im(this);
 8672   emit_int8((unsigned char)0xDD);
 8673   emit_operand32(rsi, dst);
 8674 }
 8675 
 8676 void Assembler::fnstcw(Address src) {
 8677   InstructionMark im(this);
 8678   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
 8679   emit_operand32(rdi, src);
 8680 }
 8681 
 8682 void Assembler::fnstsw_ax() {
 8683   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
 8684 }
 8685 
 8686 void Assembler::fprem() {
 8687   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
 8688 }
 8689 
 8690 void Assembler::fprem1() {
 8691   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
 8692 }
 8693 
 8694 void Assembler::frstor(Address src) {
 8695   InstructionMark im(this);
 8696   emit_int8((unsigned char)0xDD);
 8697   emit_operand32(rsp, src);
 8698 }
 8699 
 8700 void Assembler::fsin() {
 8701   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
 8702 }
 8703 
 8704 void Assembler::fsqrt() {
 8705   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
 8706 }
 8707 
 8708 void Assembler::fst_d(Address adr) {
 8709   InstructionMark im(this);
 8710   emit_int8((unsigned char)0xDD);
 8711   emit_operand32(rdx, adr);
 8712 }
 8713 
 8714 void Assembler::fst_s(Address adr) {
 8715   InstructionMark im(this);
 8716   emit_int8((unsigned char)0xD9);
 8717   emit_operand32(rdx, adr);
 8718 }
 8719 
 8720 void Assembler::fstp_d(Address adr) {
 8721   InstructionMark im(this);
 8722   emit_int8((unsigned char)0xDD);
 8723   emit_operand32(rbx, adr);
 8724 }
 8725 
 8726 void Assembler::fstp_d(int index) {
 8727   emit_farith(0xDD, 0xD8, index);
 8728 }
 8729 
 8730 void Assembler::fstp_s(Address adr) {
 8731   InstructionMark im(this);
 8732   emit_int8((unsigned char)0xD9);
 8733   emit_operand32(rbx, adr);
 8734 }
 8735 
 8736 void Assembler::fsub(int i) {
 8737   emit_farith(0xD8, 0xE0, i);
 8738 }
 8739 
 8740 void Assembler::fsub_d(Address src) {
 8741   InstructionMark im(this);
 8742   emit_int8((unsigned char)0xDC);
 8743   emit_operand32(rsp, src);
 8744 }
 8745 
 8746 void Assembler::fsub_s(Address src) {
 8747   InstructionMark im(this);
 8748   emit_int8((unsigned char)0xD8);
 8749   emit_operand32(rsp, src);
 8750 }
 8751 
 8752 void Assembler::fsuba(int i) {
 8753   emit_farith(0xDC, 0xE8, i);
 8754 }
 8755 
 8756 void Assembler::fsubp(int i) {
 8757   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
 8758 }
 8759 
 8760 void Assembler::fsubr(int i) {
 8761   emit_farith(0xD8, 0xE8, i);
 8762 }
 8763 
 8764 void Assembler::fsubr_d(Address src) {
 8765   InstructionMark im(this);
 8766   emit_int8((unsigned char)0xDC);
 8767   emit_operand32(rbp, src);
 8768 }
 8769 
 8770 void Assembler::fsubr_s(Address src) {
 8771   InstructionMark im(this);
 8772   emit_int8((unsigned char)0xD8);
 8773   emit_operand32(rbp, src);
 8774 }
 8775 
 8776 void Assembler::fsubra(int i) {
 8777   emit_farith(0xDC, 0xE0, i);
 8778 }
 8779 
 8780 void Assembler::fsubrp(int i) {
 8781   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
 8782 }
 8783 
 8784 void Assembler::ftan() {
 8785   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
 8786 }
 8787 
 8788 void Assembler::ftst() {
 8789   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
 8790 }
 8791 
 8792 void Assembler::fucomi(int i) {
 8793   // make sure the instruction is supported (introduced for P6, together with cmov)
 8794   guarantee(VM_Version::supports_cmov(), "illegal instruction");
 8795   emit_farith(0xDB, 0xE8, i);
 8796 }
 8797 
 8798 void Assembler::fucomip(int i) {
 8799   // make sure the instruction is supported (introduced for P6, together with cmov)
 8800   guarantee(VM_Version::supports_cmov(), "illegal instruction");
 8801   emit_farith(0xDF, 0xE8, i);
 8802 }
 8803 
 8804 void Assembler::fwait() {
 8805   emit_int8((unsigned char)0x9B);
 8806 }
 8807 
 8808 void Assembler::fxch(int i) {
 8809   emit_farith(0xD9, 0xC8, i);
 8810 }
 8811 
 8812 void Assembler::fyl2x() {
 8813   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
 8814 }
 8815 
 8816 void Assembler::frndint() {
 8817   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
 8818 }
 8819 
 8820 void Assembler::f2xm1() {
 8821   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
 8822 }
 8823 
 8824 void Assembler::fldl2e() {
 8825   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
 8826 }
 8827 #endif // !_LP64
 8828 
 8829 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
 8830 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
 8831 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
 8832 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
 8833 
 8834 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
 8835 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
 8836   if (pre > 0) {
 8837     emit_int8(simd_pre[pre]);
 8838   }
 8839   if (rex_w) {
 8840     prefixq(adr, xreg);
 8841   } else {
 8842     prefix(adr, xreg);
 8843   }
 8844   if (opc > 0) {
 8845     emit_int8(0x0F);
 8846     int opc2 = simd_opc[opc];
 8847     if (opc2 > 0) {
 8848       emit_int8(opc2);
 8849     }
 8850   }
 8851 }
 8852 
 8853 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
 8854   if (pre > 0) {
 8855     emit_int8(simd_pre[pre]);
 8856   }
 8857   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
 8858   if (opc > 0) {
 8859     emit_int8(0x0F);
 8860     int opc2 = simd_opc[opc];
 8861     if (opc2 > 0) {
 8862       emit_int8(opc2);
 8863     }
 8864   }
 8865   return encode;
 8866 }
 8867 
 8868 
 8869 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
 8870   int vector_len = _attributes->get_vector_len();
 8871   bool vex_w = _attributes->is_rex_vex_w();
 8872   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
 8873     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
 8874     byte1 = (~byte1) & 0xE0;
 8875     byte1 |= opc;
 8876 
 8877     int byte2 = ((~nds_enc) & 0xf) << 3;
 8878     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
 8879 
 8880     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
 8881   } else {
 8882     int byte1 = vex_r ? VEX_R : 0;
 8883     byte1 = (~byte1) & 0x80;
 8884     byte1 |= ((~nds_enc) & 0xf) << 3;
 8885     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
 8886     emit_int16((unsigned char)VEX_2bytes, byte1);
 8887   }
 8888 }
 8889 
 8890 // This is a 4 byte encoding
 8891 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
 8892   // EVEX 0x62 prefix
 8893   // byte1 = EVEX_4bytes;
 8894 
 8895   bool vex_w = _attributes->is_rex_vex_w();
 8896   int evex_encoding = (vex_w ? VEX_W : 0);
 8897   // EVEX.b is not currently used for broadcast of single element or data rounding modes
 8898   _attributes->set_evex_encoding(evex_encoding);
 8899 
 8900   // P0: byte 2, initialized to RXBR`00mm
 8901   // instead of not'd
 8902   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
 8903   byte2 = (~byte2) & 0xF0;
 8904   // confine opc opcode extensions in mm bits to lower two bits
 8905   // of form {0F, 0F_38, 0F_3A}
 8906   byte2 |= opc;
 8907 
 8908   // P1: byte 3 as Wvvvv1pp
 8909   int byte3 = ((~nds_enc) & 0xf) << 3;
 8910   // p[10] is always 1
 8911   byte3 |= EVEX_F;
 8912   byte3 |= (vex_w & 1) << 7;
 8913   // confine pre opcode extensions in pp bits to lower two bits
 8914   // of form {66, F3, F2}
 8915   byte3 |= pre;
 8916 
 8917   // P2: byte 4 as zL'Lbv'aaa
 8918   // kregs are implemented in the low 3 bits as aaa
 8919   int byte4 = (_attributes->is_no_reg_mask()) ?
 8920               0 :
 8921               _attributes->get_embedded_opmask_register_specifier();
 8922   // EVEX.v` for extending EVEX.vvvv or VIDX
 8923   byte4 |= (evex_v ? 0: EVEX_V);
 8924   // third EXEC.b for broadcast actions
 8925   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
 8926   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
 8927   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
 8928   // last is EVEX.z for zero/merge actions
 8929   if (_attributes->is_no_reg_mask() == false &&
 8930       _attributes->get_embedded_opmask_register_specifier() != 0) {
 8931     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
 8932   }
 8933 
 8934   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
 8935 }
 8936 
 8937 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
 8938   bool vex_r = (xreg_enc & 8) == 8;
 8939   bool vex_b = adr.base_needs_rex();
 8940   bool vex_x;
 8941   if (adr.isxmmindex()) {
 8942     vex_x = adr.xmmindex_needs_rex();
 8943   } else {
 8944     vex_x = adr.index_needs_rex();
 8945   }
 8946   set_attributes(attributes);
 8947   attributes->set_current_assembler(this);
 8948 
 8949   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
 8950   // is allowed in legacy mode and has resources which will fit in it.
 8951   // Pure EVEX instructions will have is_evex_instruction set in their definition.
 8952   if (!attributes->is_legacy_mode()) {
 8953     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
 8954       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
 8955           attributes->set_is_legacy_mode();
 8956       }
 8957     }
 8958   }
 8959 
 8960   if (UseAVX > 2) {
 8961     assert(((!attributes->uses_vl()) ||
 8962             (attributes->get_vector_len() == AVX_512bit) ||
 8963             (!_legacy_mode_vl) ||
 8964             (attributes->is_legacy_mode())),"XMM register should be 0-15");
 8965     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
 8966   }
 8967 
 8968   clear_managed();
 8969   if (UseAVX > 2 && !attributes->is_legacy_mode())
 8970   {
 8971     bool evex_r = (xreg_enc >= 16);
 8972     bool evex_v;
 8973     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
 8974     if (adr.isxmmindex())  {
 8975       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
 8976     } else {
 8977       evex_v = (nds_enc >= 16);
 8978     }
 8979     attributes->set_is_evex_instruction();
 8980     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
 8981   } else {
 8982     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
 8983       attributes->set_rex_vex_w(false);
 8984     }
 8985     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
 8986   }
 8987 }
 8988 
 8989 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
 8990   bool vex_r = (dst_enc & 8) == 8;
 8991   bool vex_b = (src_enc & 8) == 8;
 8992   bool vex_x = false;
 8993   set_attributes(attributes);
 8994   attributes->set_current_assembler(this);
 8995 
 8996   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
 8997   // is allowed in legacy mode and has resources which will fit in it.
 8998   // Pure EVEX instructions will have is_evex_instruction set in their definition.
 8999   if (!attributes->is_legacy_mode()) {
 9000     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
 9001       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
 9002           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
 9003           attributes->set_is_legacy_mode();
 9004       }
 9005     }
 9006   }
 9007 
 9008   if (UseAVX > 2) {
 9009     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
 9010     // Instruction with uses_vl true are vector instructions
 9011     // All the vector instructions with AVX_512bit length can have legacy_mode as false
 9012     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
 9013     // Rest all should have legacy_mode set as true
 9014     assert(((!attributes->uses_vl()) ||
 9015             (attributes->get_vector_len() == AVX_512bit) ||
 9016             (!_legacy_mode_vl) ||
 9017             (attributes->is_legacy_mode())),"XMM register should be 0-15");
 9018     // Instruction with legacy_mode true should have dst, nds and src < 15
 9019     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
 9020   }
 9021 
 9022   clear_managed();
 9023   if (UseAVX > 2 && !attributes->is_legacy_mode())
 9024   {
 9025     bool evex_r = (dst_enc >= 16);
 9026     bool evex_v = (nds_enc >= 16);
 9027     // can use vex_x as bank extender on rm encoding
 9028     vex_x = (src_enc >= 16);
 9029     attributes->set_is_evex_instruction();
 9030     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
 9031   } else {
 9032     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
 9033       attributes->set_rex_vex_w(false);
 9034     }
 9035     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
 9036   }
 9037 
 9038   // return modrm byte components for operands
 9039   return (((dst_enc & 7) << 3) | (src_enc & 7));
 9040 }
 9041 
 9042 
 9043 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
 9044                             VexOpcode opc, InstructionAttr *attributes) {
 9045   if (UseAVX > 0) {
 9046     int xreg_enc = xreg->encoding();
 9047     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
 9048     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
 9049   } else {
 9050     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
 9051     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
 9052   }
 9053 }
 9054 
 9055 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
 9056                                       VexOpcode opc, InstructionAttr *attributes) {
 9057   int dst_enc = dst->encoding();
 9058   int src_enc = src->encoding();
 9059   if (UseAVX > 0) {
 9060     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
 9061     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
 9062   } else {
 9063     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
 9064     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
 9065   }
 9066 }
 9067 
 9068 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9069   assert(VM_Version::supports_avx(), "");
 9070   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9071   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 9072   emit_int16(0x5F, (0xC0 | encode));
 9073 }
 9074 
 9075 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9076   assert(VM_Version::supports_avx(), "");
 9077   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9078   attributes.set_rex_vex_w_reverted();
 9079   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 9080   emit_int16(0x5F, (0xC0 | encode));
 9081 }
 9082 
 9083 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9084   assert(VM_Version::supports_avx(), "");
 9085   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9086   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 9087   emit_int16(0x5D, (0xC0 | encode));
 9088 }
 9089 
 9090 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 9091   assert(VM_Version::supports_avx(), "");
 9092   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9093   attributes.set_rex_vex_w_reverted();
 9094   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 9095   emit_int16(0x5D, (0xC0 | encode));
 9096 }
 9097 
 9098 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
 9099   assert(VM_Version::supports_avx(), "");
 9100   assert(vector_len <= AVX_256bit, "");
 9101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9102   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9103   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
 9104 }
 9105 
 9106 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9107   assert(VM_Version::supports_avx(), "");
 9108   assert(vector_len <= AVX_256bit, "");
 9109   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9110   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9111   int src2_enc = src2->encoding();
 9112   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9113 }
 9114 
 9115 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9116   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
 9117   assert(vector_len <= AVX_256bit, "");
 9118   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9119   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9120   int src2_enc = src2->encoding();
 9121   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9122 }
 9123 
 9124 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 9125   assert(VM_Version::supports_avx2(), "");
 9126   assert(vector_len <= AVX_256bit, "");
 9127   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9128   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9129   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
 9130 }
 9131 
 9132 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
 9133   assert(VM_Version::supports_avx(), "");
 9134   assert(vector_len <= AVX_256bit, "");
 9135   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9136   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9137   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
 9138 }
 9139 
 9140 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9141                         ComparisonPredicateFP comparison, int vector_len) {
 9142   assert(VM_Version::supports_evex(), "");
 9143   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
 9144   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9145   attributes.set_is_evex_instruction();
 9146   attributes.set_embedded_opmask_register_specifier(mask);
 9147   attributes.reset_is_clear_context();
 9148   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9149   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
 9150 }
 9151 
 9152 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9153                         ComparisonPredicateFP comparison, int vector_len) {
 9154   assert(VM_Version::supports_evex(), "");
 9155   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
 9156   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9157   attributes.set_is_evex_instruction();
 9158   attributes.set_embedded_opmask_register_specifier(mask);
 9159   attributes.reset_is_clear_context();
 9160   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9161   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
 9162 }
 9163 
 9164 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
 9165   assert(VM_Version::supports_sse4_1(), "");
 9166   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9167   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9169   emit_int16(0x14, (0xC0 | encode));
 9170 }
 9171 
 9172 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
 9173   assert(VM_Version::supports_sse4_1(), "");
 9174   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9176   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9177   emit_int16(0x15, (0xC0 | encode));
 9178 }
 9179 
 9180 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
 9181   assert(VM_Version::supports_sse4_1(), "");
 9182   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
 9183   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9184   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9185   emit_int16(0x10, (0xC0 | encode));
 9186 }
 9187 
 9188 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
 9189   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
 9190   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9191   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9192   int src2_enc = src2->encoding();
 9193   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
 9194 }
 9195 
 9196 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 9197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9198   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9199   emit_int24(0x0C, (0xC0 | encode), imm8);
 9200 }
 9201 
 9202 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9203   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9204   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9206   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9207   emit_int16(0x64, (0xC0 | encode));
 9208 }
 9209 
 9210 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9211   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9212   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9213   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9214   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9215   emit_int16(0x65, (0xC0 | encode));
 9216 }
 9217 
 9218 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9219   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9220   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9222   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9223   emit_int16(0x66, (0xC0 | encode));
 9224 }
 9225 
 9226 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9227   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 9228   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 9229   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9230   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9231   emit_int16(0x37, (0xC0 | encode));
 9232 }
 9233 
 9234 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9235                         int comparison, bool is_signed, int vector_len) {
 9236   assert(VM_Version::supports_evex(), "");
 9237   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9238   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
 9239   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9240   attributes.set_is_evex_instruction();
 9241   attributes.set_embedded_opmask_register_specifier(mask);
 9242   attributes.reset_is_clear_context();
 9243   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9244   int opcode = is_signed ? 0x1F : 0x1E;
 9245   emit_int24(opcode, (0xC0 | encode), comparison);
 9246 }
 9247 
 9248 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9249                         int comparison, bool is_signed, int vector_len) {
 9250   assert(VM_Version::supports_evex(), "");
 9251   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9252   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
 9253   InstructionMark im(this);
 9254   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9255   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 9256   attributes.set_is_evex_instruction();
 9257   attributes.set_embedded_opmask_register_specifier(mask);
 9258   attributes.reset_is_clear_context();
 9259   int dst_enc = kdst->encoding();
 9260   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9261   int opcode = is_signed ? 0x1F : 0x1E;
 9262   emit_int8((unsigned char)opcode);
 9263   emit_operand(as_Register(dst_enc), src);
 9264   emit_int8((unsigned char)comparison);
 9265 }
 9266 
 9267 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9268                         int comparison, bool is_signed, int vector_len) {
 9269   assert(VM_Version::supports_evex(), "");
 9270   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9271   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
 9272   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9273   attributes.set_is_evex_instruction();
 9274   attributes.set_embedded_opmask_register_specifier(mask);
 9275   attributes.reset_is_clear_context();
 9276   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9277   int opcode = is_signed ? 0x1F : 0x1E;
 9278   emit_int24(opcode, (0xC0 | encode), comparison);
 9279 }
 9280 
 9281 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9282                         int comparison, bool is_signed, int vector_len) {
 9283   assert(VM_Version::supports_evex(), "");
 9284   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9285   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
 9286   InstructionMark im(this);
 9287   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9288   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 9289   attributes.set_is_evex_instruction();
 9290   attributes.set_embedded_opmask_register_specifier(mask);
 9291   attributes.reset_is_clear_context();
 9292   int dst_enc = kdst->encoding();
 9293   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9294   int opcode = is_signed ? 0x1F : 0x1E;
 9295   emit_int8((unsigned char)opcode);
 9296   emit_operand(as_Register(dst_enc), src);
 9297   emit_int8((unsigned char)comparison);
 9298 }
 9299 
 9300 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9301                         int comparison, bool is_signed, int vector_len) {
 9302   assert(VM_Version::supports_evex(), "");
 9303   assert(VM_Version::supports_avx512bw(), "");
 9304   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9305   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
 9306   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9307   attributes.set_is_evex_instruction();
 9308   attributes.set_embedded_opmask_register_specifier(mask);
 9309   attributes.reset_is_clear_context();
 9310   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9311   int opcode = is_signed ? 0x3F : 0x3E;
 9312   emit_int24(opcode, (0xC0 | encode), comparison);
 9313 }
 9314 
 9315 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9316                         int comparison, bool is_signed, int vector_len) {
 9317   assert(VM_Version::supports_evex(), "");
 9318   assert(VM_Version::supports_avx512bw(), "");
 9319   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9320   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
 9321   InstructionMark im(this);
 9322   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9323   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 9324   attributes.set_is_evex_instruction();
 9325   attributes.set_embedded_opmask_register_specifier(mask);
 9326   attributes.reset_is_clear_context();
 9327   int dst_enc = kdst->encoding();
 9328   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9329   int opcode = is_signed ? 0x3F : 0x3E;
 9330   emit_int8((unsigned char)opcode);
 9331   emit_operand(as_Register(dst_enc), src);
 9332   emit_int8((unsigned char)comparison);
 9333 }
 9334 
 9335 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
 9336                         int comparison, bool is_signed, int vector_len) {
 9337   assert(VM_Version::supports_evex(), "");
 9338   assert(VM_Version::supports_avx512bw(), "");
 9339   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9340   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
 9341   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9342   attributes.set_is_evex_instruction();
 9343   attributes.set_embedded_opmask_register_specifier(mask);
 9344   attributes.reset_is_clear_context();
 9345   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9346   int opcode = is_signed ? 0x3F : 0x3E;
 9347   emit_int24(opcode, (0xC0 | encode), comparison);
 9348 }
 9349 
 9350 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
 9351                         int comparison, bool is_signed, int vector_len) {
 9352   assert(VM_Version::supports_evex(), "");
 9353   assert(VM_Version::supports_avx512bw(), "");
 9354   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
 9355   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
 9356   InstructionMark im(this);
 9357   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9358   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 9359   attributes.set_is_evex_instruction();
 9360   attributes.set_embedded_opmask_register_specifier(mask);
 9361   attributes.reset_is_clear_context();
 9362   int dst_enc = kdst->encoding();
 9363   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9364   int opcode = is_signed ? 0x3F : 0x3E;
 9365   emit_int8((unsigned char)opcode);
 9366   emit_operand(as_Register(dst_enc), src);
 9367   emit_int8((unsigned char)comparison);
 9368 }
 9369 
 9370 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
 9371   assert(VM_Version::supports_avx(), "");
 9372   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9373   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 9374   int mask_enc = mask->encoding();
 9375   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
 9376 }
 9377 
 9378 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9379   assert(VM_Version::supports_evex(), "");
 9380   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
 9381   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9382   attributes.set_is_evex_instruction();
 9383   attributes.set_embedded_opmask_register_specifier(mask);
 9384   if (merge) {
 9385     attributes.reset_is_clear_context();
 9386   }
 9387   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9388   emit_int16(0x65, (0xC0 | encode));
 9389 }
 9390 
 9391 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9392   assert(VM_Version::supports_evex(), "");
 9393   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
 9394   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9395   attributes.set_is_evex_instruction();
 9396   attributes.set_embedded_opmask_register_specifier(mask);
 9397   if (merge) {
 9398     attributes.reset_is_clear_context();
 9399   }
 9400   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9401   emit_int16(0x65, (0xC0 | encode));
 9402 }
 9403 
 9404 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9405   assert(VM_Version::supports_evex(), "");
 9406   assert(VM_Version::supports_avx512bw(), "");
 9407   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
 9408   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9409   attributes.set_is_evex_instruction();
 9410   attributes.set_embedded_opmask_register_specifier(mask);
 9411   if (merge) {
 9412     attributes.reset_is_clear_context();
 9413   }
 9414   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9415   emit_int16(0x66, (0xC0 | encode));
 9416 }
 9417 
 9418 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9419   assert(VM_Version::supports_evex(), "");
 9420   assert(VM_Version::supports_avx512bw(), "");
 9421   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
 9422   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 9423   attributes.set_is_evex_instruction();
 9424   attributes.set_embedded_opmask_register_specifier(mask);
 9425   if (merge) {
 9426     attributes.reset_is_clear_context();
 9427   }
 9428   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9429   emit_int16(0x66, (0xC0 | encode));
 9430 }
 9431 
 9432 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9433   assert(VM_Version::supports_evex(), "");
 9434   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
 9435   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9436   attributes.set_is_evex_instruction();
 9437   attributes.set_embedded_opmask_register_specifier(mask);
 9438   if (merge) {
 9439     attributes.reset_is_clear_context();
 9440   }
 9441   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9442   emit_int16(0x64, (0xC0 | encode));
 9443 }
 9444 
 9445 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9446   assert(VM_Version::supports_evex(), "");
 9447   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
 9448   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9449   attributes.set_is_evex_instruction();
 9450   attributes.set_embedded_opmask_register_specifier(mask);
 9451   if (merge) {
 9452     attributes.reset_is_clear_context();
 9453   }
 9454   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9455   emit_int16(0x64, (0xC0 | encode));
 9456 }
 9457 
 9458 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
 9459   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
 9460   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 9461   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 9462   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 9463 }
 9464 
 9465 void Assembler::shlxl(Register dst, Register src1, Register src2) {
 9466   assert(VM_Version::supports_bmi2(), "");
 9467   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9468   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9469   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9470 }
 9471 
 9472 void Assembler::shlxq(Register dst, Register src1, Register src2) {
 9473   assert(VM_Version::supports_bmi2(), "");
 9474   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9475   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9476   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9477 }
 9478 
 9479 void Assembler::shrxl(Register dst, Register src1, Register src2) {
 9480   assert(VM_Version::supports_bmi2(), "");
 9481   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9482   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
 9483   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9484 }
 9485 
 9486 void Assembler::shrxq(Register dst, Register src1, Register src2) {
 9487   assert(VM_Version::supports_bmi2(), "");
 9488   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 9489   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
 9490   emit_int16((unsigned char)0xF7, (0xC0 | encode));
 9491 }
 9492 
 9493 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
 9494   assert(VM_Version::supports_avx512vlbw(), "");
 9495   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9496   attributes.set_is_evex_instruction();
 9497   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 9498   emit_int16(0x29, (0xC0 | encode));
 9499 }
 9500 
 9501 #ifndef _LP64
 9502 
 9503 void Assembler::incl(Register dst) {
 9504   // Don't use it directly. Use MacroAssembler::incrementl() instead.
 9505   emit_int8(0x40 | dst->encoding());
 9506 }
 9507 
 9508 void Assembler::lea(Register dst, Address src) {
 9509   leal(dst, src);
 9510 }
 9511 
 9512 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
 9513   InstructionMark im(this);
 9514   emit_int8((unsigned char)0xC7);
 9515   emit_operand(rax, dst);
 9516   emit_data((int)imm32, rspec, 0);
 9517 }
 9518 
 9519 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
 9520   InstructionMark im(this);
 9521   int encode = prefix_and_encode(dst->encoding());
 9522   emit_int8((0xB8 | encode));
 9523   emit_data((int)imm32, rspec, 0);
 9524 }
 9525 
 9526 void Assembler::popa() { // 32bit
 9527   emit_int8(0x61);
 9528 }
 9529 
 9530 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
 9531   InstructionMark im(this);
 9532   emit_int8(0x68);
 9533   emit_data(imm32, rspec, 0);
 9534 }
 9535 
 9536 void Assembler::pusha() { // 32bit
 9537   emit_int8(0x60);
 9538 }
 9539 
 9540 void Assembler::set_byte_if_not_zero(Register dst) {
 9541   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
 9542 }
 9543 
 9544 #else // LP64
 9545 
 9546 void Assembler::set_byte_if_not_zero(Register dst) {
 9547   int enc = prefix_and_encode(dst->encoding(), true);
 9548   emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
 9549 }
 9550 
 9551 // 64bit only pieces of the assembler
 9552 // This should only be used by 64bit instructions that can use rip-relative
 9553 // it cannot be used by instructions that want an immediate value.
 9554 
 9555 bool Assembler::reachable(AddressLiteral adr) {
 9556   int64_t disp;
 9557   relocInfo::relocType relocType = adr.reloc();
 9558 
 9559   // None will force a 64bit literal to the code stream. Likely a placeholder
 9560   // for something that will be patched later and we need to certain it will
 9561   // always be reachable.
 9562   if (relocType == relocInfo::none) {
 9563     return false;
 9564   }
 9565   if (relocType == relocInfo::internal_word_type) {
 9566     // This should be rip relative and easily reachable.
 9567     return true;
 9568   }
 9569   if (relocType == relocInfo::virtual_call_type ||
 9570       relocType == relocInfo::opt_virtual_call_type ||
 9571       relocType == relocInfo::static_call_type ||
 9572       relocType == relocInfo::static_stub_type ) {
 9573     // This should be rip relative within the code cache and easily
 9574     // reachable until we get huge code caches. (At which point
 9575     // ic code is going to have issues).
 9576     return true;
 9577   }
 9578   if (relocType != relocInfo::external_word_type &&
 9579       relocType != relocInfo::poll_return_type &&  // these are really external_word but need special
 9580       relocType != relocInfo::poll_type &&         // relocs to identify them
 9581       relocType != relocInfo::runtime_call_type ) {
 9582     return false;
 9583   }
 9584 
 9585   // Stress the correction code
 9586   if (ForceUnreachable) {
 9587     // Must be runtimecall reloc, see if it is in the codecache
 9588     // Flipping stuff in the codecache to be unreachable causes issues
 9589     // with things like inline caches where the additional instructions
 9590     // are not handled.
 9591     if (CodeCache::find_blob(adr._target) == NULL) {
 9592       return false;
 9593     }
 9594   }
 9595   // For external_word_type/runtime_call_type if it is reachable from where we
 9596   // are now (possibly a temp buffer) and where we might end up
 9597   // anywhere in the codeCache then we are always reachable.
 9598   // This would have to change if we ever save/restore shared code
 9599   // to be more pessimistic.
 9600   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
 9601   if (!is_simm32(disp)) return false;
 9602   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
 9603   if (!is_simm32(disp)) return false;
 9604 
 9605   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
 9606 
 9607   // Because rip relative is a disp + address_of_next_instruction and we
 9608   // don't know the value of address_of_next_instruction we apply a fudge factor
 9609   // to make sure we will be ok no matter the size of the instruction we get placed into.
 9610   // We don't have to fudge the checks above here because they are already worst case.
 9611 
 9612   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
 9613   // + 4 because better safe than sorry.
 9614   const int fudge = 12 + 4;
 9615   if (disp < 0) {
 9616     disp -= fudge;
 9617   } else {
 9618     disp += fudge;
 9619   }
 9620   return is_simm32(disp);
 9621 }
 9622 
 9623 void Assembler::emit_data64(jlong data,
 9624                             relocInfo::relocType rtype,
 9625                             int format) {
 9626   if (rtype == relocInfo::none) {
 9627     emit_int64(data);
 9628   } else {
 9629     emit_data64(data, Relocation::spec_simple(rtype), format);
 9630   }
 9631 }
 9632 
 9633 void Assembler::emit_data64(jlong data,
 9634                             RelocationHolder const& rspec,
 9635                             int format) {
 9636   assert(imm_operand == 0, "default format must be immediate in this file");
 9637   assert(imm_operand == format, "must be immediate");
 9638   assert(inst_mark() != NULL, "must be inside InstructionMark");
 9639   // Do not use AbstractAssembler::relocate, which is not intended for
 9640   // embedded words.  Instead, relocate to the enclosing instruction.
 9641   code_section()->relocate(inst_mark(), rspec, format);
 9642 #ifdef ASSERT
 9643   check_relocation(rspec, format);
 9644 #endif
 9645   emit_int64(data);
 9646 }
 9647 
 9648 void Assembler::prefix(Register reg) {
 9649   if (reg->encoding() >= 8) {
 9650     prefix(REX_B);
 9651   }
 9652 }
 9653 
 9654 void Assembler::prefix(Register dst, Register src, Prefix p) {
 9655   if (src->encoding() >= 8) {
 9656     p = (Prefix)(p | REX_B);
 9657   }
 9658   if (dst->encoding() >= 8) {
 9659     p = (Prefix)(p | REX_R);
 9660   }
 9661   if (p != Prefix_EMPTY) {
 9662     // do not generate an empty prefix
 9663     prefix(p);
 9664   }
 9665 }
 9666 
 9667 void Assembler::prefix(Register dst, Address adr, Prefix p) {
 9668   if (adr.base_needs_rex()) {
 9669     if (adr.index_needs_rex()) {
 9670       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
 9671     } else {
 9672       prefix(REX_B);
 9673     }
 9674   } else {
 9675     if (adr.index_needs_rex()) {
 9676       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
 9677     }
 9678   }
 9679   if (dst->encoding() >= 8) {
 9680     p = (Prefix)(p | REX_R);
 9681   }
 9682   if (p != Prefix_EMPTY) {
 9683     // do not generate an empty prefix
 9684     prefix(p);
 9685   }
 9686 }
 9687 
 9688 void Assembler::prefix(Address adr) {
 9689   if (adr.base_needs_rex()) {
 9690     if (adr.index_needs_rex()) {
 9691       prefix(REX_XB);
 9692     } else {
 9693       prefix(REX_B);
 9694     }
 9695   } else {
 9696     if (adr.index_needs_rex()) {
 9697       prefix(REX_X);
 9698     }
 9699   }
 9700 }
 9701 
 9702 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
 9703   if (reg->encoding() < 8) {
 9704     if (adr.base_needs_rex()) {
 9705       if (adr.index_needs_rex()) {
 9706         prefix(REX_XB);
 9707       } else {
 9708         prefix(REX_B);
 9709       }
 9710     } else {
 9711       if (adr.index_needs_rex()) {
 9712         prefix(REX_X);
 9713       } else if (byteinst && reg->encoding() >= 4) {
 9714         prefix(REX);
 9715       }
 9716     }
 9717   } else {
 9718     if (adr.base_needs_rex()) {
 9719       if (adr.index_needs_rex()) {
 9720         prefix(REX_RXB);
 9721       } else {
 9722         prefix(REX_RB);
 9723       }
 9724     } else {
 9725       if (adr.index_needs_rex()) {
 9726         prefix(REX_RX);
 9727       } else {
 9728         prefix(REX_R);
 9729       }
 9730     }
 9731   }
 9732 }
 9733 
 9734 void Assembler::prefix(Address adr, XMMRegister reg) {
 9735   if (reg->encoding() < 8) {
 9736     if (adr.base_needs_rex()) {
 9737       if (adr.index_needs_rex()) {
 9738         prefix(REX_XB);
 9739       } else {
 9740         prefix(REX_B);
 9741       }
 9742     } else {
 9743       if (adr.index_needs_rex()) {
 9744         prefix(REX_X);
 9745       }
 9746     }
 9747   } else {
 9748     if (adr.base_needs_rex()) {
 9749       if (adr.index_needs_rex()) {
 9750         prefix(REX_RXB);
 9751       } else {
 9752         prefix(REX_RB);
 9753       }
 9754     } else {
 9755       if (adr.index_needs_rex()) {
 9756         prefix(REX_RX);
 9757       } else {
 9758         prefix(REX_R);
 9759       }
 9760     }
 9761   }
 9762 }
 9763 
 9764 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
 9765   if (reg_enc >= 8) {
 9766     prefix(REX_B);
 9767     reg_enc -= 8;
 9768   } else if (byteinst && reg_enc >= 4) {
 9769     prefix(REX);
 9770   }
 9771   return reg_enc;
 9772 }
 9773 
 9774 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
 9775   if (dst_enc < 8) {
 9776     if (src_enc >= 8) {
 9777       prefix(REX_B);
 9778       src_enc -= 8;
 9779     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
 9780       prefix(REX);
 9781     }
 9782   } else {
 9783     if (src_enc < 8) {
 9784       prefix(REX_R);
 9785     } else {
 9786       prefix(REX_RB);
 9787       src_enc -= 8;
 9788     }
 9789     dst_enc -= 8;
 9790   }
 9791   return dst_enc << 3 | src_enc;
 9792 }
 9793 
 9794 int8_t Assembler::get_prefixq(Address adr) {
 9795   int8_t prfx = get_prefixq(adr, rax);
 9796   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
 9797   return prfx;
 9798 }
 9799 
 9800 int8_t Assembler::get_prefixq(Address adr, Register src) {
 9801   int8_t prfx = (int8_t)(REX_W +
 9802                          ((int)adr.base_needs_rex()) +
 9803                          ((int)adr.index_needs_rex() << 1) +
 9804                          ((int)(src->encoding() >= 8) << 2));
 9805 #ifdef ASSERT
 9806   if (src->encoding() < 8) {
 9807     if (adr.base_needs_rex()) {
 9808       if (adr.index_needs_rex()) {
 9809         assert(prfx == REX_WXB, "must be");
 9810       } else {
 9811         assert(prfx == REX_WB, "must be");
 9812       }
 9813     } else {
 9814       if (adr.index_needs_rex()) {
 9815         assert(prfx == REX_WX, "must be");
 9816       } else {
 9817         assert(prfx == REX_W, "must be");
 9818       }
 9819     }
 9820   } else {
 9821     if (adr.base_needs_rex()) {
 9822       if (adr.index_needs_rex()) {
 9823         assert(prfx == REX_WRXB, "must be");
 9824       } else {
 9825         assert(prfx == REX_WRB, "must be");
 9826       }
 9827     } else {
 9828       if (adr.index_needs_rex()) {
 9829         assert(prfx == REX_WRX, "must be");
 9830       } else {
 9831         assert(prfx == REX_WR, "must be");
 9832       }
 9833     }
 9834   }
 9835 #endif
 9836   return prfx;
 9837 }
 9838 
 9839 void Assembler::prefixq(Address adr) {
 9840   emit_int8(get_prefixq(adr));
 9841 }
 9842 
 9843 void Assembler::prefixq(Address adr, Register src) {
 9844   emit_int8(get_prefixq(adr, src));
 9845 }
 9846 
 9847 void Assembler::prefixq(Address adr, XMMRegister src) {
 9848   if (src->encoding() < 8) {
 9849     if (adr.base_needs_rex()) {
 9850       if (adr.index_needs_rex()) {
 9851         prefix(REX_WXB);
 9852       } else {
 9853         prefix(REX_WB);
 9854       }
 9855     } else {
 9856       if (adr.index_needs_rex()) {
 9857         prefix(REX_WX);
 9858       } else {
 9859         prefix(REX_W);
 9860       }
 9861     }
 9862   } else {
 9863     if (adr.base_needs_rex()) {
 9864       if (adr.index_needs_rex()) {
 9865         prefix(REX_WRXB);
 9866       } else {
 9867         prefix(REX_WRB);
 9868       }
 9869     } else {
 9870       if (adr.index_needs_rex()) {
 9871         prefix(REX_WRX);
 9872       } else {
 9873         prefix(REX_WR);
 9874       }
 9875     }
 9876   }
 9877 }
 9878 
 9879 int Assembler::prefixq_and_encode(int reg_enc) {
 9880   if (reg_enc < 8) {
 9881     prefix(REX_W);
 9882   } else {
 9883     prefix(REX_WB);
 9884     reg_enc -= 8;
 9885   }
 9886   return reg_enc;
 9887 }
 9888 
 9889 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
 9890   if (dst_enc < 8) {
 9891     if (src_enc < 8) {
 9892       prefix(REX_W);
 9893     } else {
 9894       prefix(REX_WB);
 9895       src_enc -= 8;
 9896     }
 9897   } else {
 9898     if (src_enc < 8) {
 9899       prefix(REX_WR);
 9900     } else {
 9901       prefix(REX_WRB);
 9902       src_enc -= 8;
 9903     }
 9904     dst_enc -= 8;
 9905   }
 9906   return dst_enc << 3 | src_enc;
 9907 }
 9908 
 9909 void Assembler::adcq(Register dst, int32_t imm32) {
 9910   (void) prefixq_and_encode(dst->encoding());
 9911   emit_arith(0x81, 0xD0, dst, imm32);
 9912 }
 9913 
 9914 void Assembler::adcq(Register dst, Address src) {
 9915   InstructionMark im(this);
 9916   emit_int16(get_prefixq(src, dst), 0x13);
 9917   emit_operand(dst, src);
 9918 }
 9919 
 9920 void Assembler::adcq(Register dst, Register src) {
 9921   (void) prefixq_and_encode(dst->encoding(), src->encoding());
 9922   emit_arith(0x13, 0xC0, dst, src);
 9923 }
 9924 
 9925 void Assembler::addq(Address dst, int32_t imm32) {
 9926   InstructionMark im(this);
 9927   prefixq(dst);
 9928   emit_arith_operand(0x81, rax, dst, imm32);
 9929 }
 9930 
 9931 void Assembler::addq(Address dst, Register src) {
 9932   InstructionMark im(this);
 9933   emit_int16(get_prefixq(dst, src), 0x01);
 9934   emit_operand(src, dst);
 9935 }
 9936 
 9937 void Assembler::addq(Register dst, int32_t imm32) {
 9938   (void) prefixq_and_encode(dst->encoding());
 9939   emit_arith(0x81, 0xC0, dst, imm32);
 9940 }
 9941 
 9942 void Assembler::addq(Register dst, Address src) {
 9943   InstructionMark im(this);
 9944   emit_int16(get_prefixq(src, dst), 0x03);
 9945   emit_operand(dst, src);
 9946 }
 9947 
 9948 void Assembler::addq(Register dst, Register src) {
 9949   (void) prefixq_and_encode(dst->encoding(), src->encoding());
 9950   emit_arith(0x03, 0xC0, dst, src);
 9951 }
 9952 
 9953 void Assembler::adcxq(Register dst, Register src) {
 9954   //assert(VM_Version::supports_adx(), "adx instructions not supported");
 9955   emit_int8(0x66);
 9956   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 9957   emit_int32(0x0F,
 9958              0x38,
 9959              (unsigned char)0xF6,
 9960              (0xC0 | encode));
 9961 }
 9962 
 9963 void Assembler::adoxq(Register dst, Register src) {
 9964   //assert(VM_Version::supports_adx(), "adx instructions not supported");
 9965   emit_int8((unsigned char)0xF3);
 9966   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 9967   emit_int32(0x0F,
 9968              0x38,
 9969              (unsigned char)0xF6,
 9970              (0xC0 | encode));
 9971 }
 9972 
 9973 void Assembler::andq(Address dst, int32_t imm32) {
 9974   InstructionMark im(this);
 9975   prefixq(dst);
 9976   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 9977 }
 9978 
 9979 void Assembler::andq(Register dst, int32_t imm32) {
 9980   (void) prefixq_and_encode(dst->encoding());
 9981   emit_arith(0x81, 0xE0, dst, imm32);
 9982 }
 9983 
 9984 void Assembler::andq(Register dst, Address src) {
 9985   InstructionMark im(this);
 9986   emit_int16(get_prefixq(src, dst), 0x23);
 9987   emit_operand(dst, src);
 9988 }
 9989 
 9990 void Assembler::andq(Register dst, Register src) {
 9991   (void) prefixq_and_encode(dst->encoding(), src->encoding());
 9992   emit_arith(0x23, 0xC0, dst, src);
 9993 }
 9994 
 9995 void Assembler::andq(Address dst, Register src) {
 9996   InstructionMark im(this);
 9997   emit_int16(get_prefixq(dst, src), 0x21);
 9998   emit_operand(src, dst);
 9999 }
10000 
10001 void Assembler::andnq(Register dst, Register src1, Register src2) {
10002   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10003   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10004   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10005   emit_int16((unsigned char)0xF2, (0xC0 | encode));
10006 }
10007 
10008 void Assembler::andnq(Register dst, Register src1, Address src2) {
10009   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10010   InstructionMark im(this);
10011   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10012   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10013   emit_int8((unsigned char)0xF2);
10014   emit_operand(dst, src2);
10015 }
10016 
10017 void Assembler::bsfq(Register dst, Register src) {
10018   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10019   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
10020 }
10021 
10022 void Assembler::bsrq(Register dst, Register src) {
10023   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10024   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
10025 }
10026 
10027 void Assembler::bswapq(Register reg) {
10028   int encode = prefixq_and_encode(reg->encoding());
10029   emit_int16(0x0F, (0xC8 | encode));
10030 }
10031 
10032 void Assembler::blsiq(Register dst, Register src) {
10033   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10034   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10035   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10036   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10037 }
10038 
10039 void Assembler::blsiq(Register dst, Address src) {
10040   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10041   InstructionMark im(this);
10042   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10043   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10044   emit_int8((unsigned char)0xF3);
10045   emit_operand(rbx, src);
10046 }
10047 
10048 void Assembler::blsmskq(Register dst, Register src) {
10049   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10050   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10051   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10052   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10053 }
10054 
10055 void Assembler::blsmskq(Register dst, Address src) {
10056   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10057   InstructionMark im(this);
10058   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10059   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10060   emit_int8((unsigned char)0xF3);
10061   emit_operand(rdx, src);
10062 }
10063 
10064 void Assembler::blsrq(Register dst, Register src) {
10065   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10066   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10067   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10068   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10069 }
10070 
10071 void Assembler::blsrq(Register dst, Address src) {
10072   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
10073   InstructionMark im(this);
10074   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10075   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
10076   emit_int8((unsigned char)0xF3);
10077   emit_operand(rcx, src);
10078 }
10079 
10080 void Assembler::cdqq() {
10081   emit_int16(REX_W, (unsigned char)0x99);
10082 }
10083 
10084 void Assembler::clflush(Address adr) {
10085   assert(VM_Version::supports_clflush(), "should do");
10086   prefix(adr);
10087   emit_int16(0x0F, (unsigned char)0xAE);
10088   emit_operand(rdi, adr);
10089 }
10090 
10091 void Assembler::clflushopt(Address adr) {
10092   assert(VM_Version::supports_clflushopt(), "should do!");
10093   // adr should be base reg only with no index or offset
10094   assert(adr.index() == noreg, "index should be noreg");
10095   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
10096   assert(adr.disp() == 0, "displacement should be 0");
10097   // instruction prefix is 0x66
10098   emit_int8(0x66);
10099   prefix(adr);
10100   // opcode family is 0x0F 0xAE
10101   emit_int16(0x0F, (unsigned char)0xAE);
10102   // extended opcode byte is 7 == rdi
10103   emit_operand(rdi, adr);
10104 }
10105 
10106 void Assembler::clwb(Address adr) {
10107   assert(VM_Version::supports_clwb(), "should do!");
10108   // adr should be base reg only with no index or offset
10109   assert(adr.index() == noreg, "index should be noreg");
10110   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
10111   assert(adr.disp() == 0, "displacement should be 0");
10112   // instruction prefix is 0x66
10113   emit_int8(0x66);
10114   prefix(adr);
10115   // opcode family is 0x0f 0xAE
10116   emit_int16(0x0F, (unsigned char)0xAE);
10117   // extended opcode byte is 6 == rsi
10118   emit_operand(rsi, adr);
10119 }
10120 
10121 void Assembler::cmovq(Condition cc, Register dst, Register src) {
10122   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10123   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
10124 }
10125 
10126 void Assembler::cmovq(Condition cc, Register dst, Address src) {
10127   InstructionMark im(this);
10128   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
10129   emit_operand(dst, src);
10130 }
10131 
10132 void Assembler::cmpq(Address dst, int32_t imm32) {
10133   InstructionMark im(this);
10134   emit_int16(get_prefixq(dst), (unsigned char)0x81);
10135   emit_operand(rdi, dst, 4);
10136   emit_int32(imm32);
10137 }
10138 
10139 void Assembler::cmpq(Register dst, int32_t imm32) {
10140   (void) prefixq_and_encode(dst->encoding());
10141   emit_arith(0x81, 0xF8, dst, imm32);
10142 }
10143 
10144 void Assembler::cmpq(Address dst, Register src) {
10145   InstructionMark im(this);
10146   emit_int16(get_prefixq(dst, src), 0x39);
10147   emit_operand(src, dst);
10148 }
10149 
10150 void Assembler::cmpq(Register dst, Register src) {
10151   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10152   emit_arith(0x3B, 0xC0, dst, src);
10153 }
10154 
10155 void Assembler::cmpq(Register dst, Address src) {
10156   InstructionMark im(this);
10157   emit_int16(get_prefixq(src, dst), 0x3B);
10158   emit_operand(dst, src);
10159 }
10160 
10161 void Assembler::cmpxchgq(Register reg, Address adr) {
10162   InstructionMark im(this);
10163   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
10164   emit_operand(reg, adr);
10165 }
10166 
10167 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
10168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10169   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10170   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10171   emit_int16(0x2A, (0xC0 | encode));
10172 }
10173 
10174 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
10175   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10176   InstructionMark im(this);
10177   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10178   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10179   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10180   emit_int8(0x2A);
10181   emit_operand(dst, src);
10182 }
10183 
10184 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
10185   NOT_LP64(assert(VM_Version::supports_sse(), ""));
10186   InstructionMark im(this);
10187   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10188   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10189   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10190   emit_int8(0x2A);
10191   emit_operand(dst, src);
10192 }
10193 
10194 void Assembler::cvttsd2siq(Register dst, Address src) {
10195   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10196   // F2 REX.W 0F 2C /r
10197   // CVTTSD2SI r64, xmm1/m64
10198   InstructionMark im(this);
10199   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
10200   emit_operand(dst, src);
10201 }
10202 
10203 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
10204   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10205   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10206   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
10207   emit_int16(0x2C, (0xC0 | encode));
10208 }
10209 
10210 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
10211   NOT_LP64(assert(VM_Version::supports_sse(), ""));
10212   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10213   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
10214   emit_int16(0x2C, (0xC0 | encode));
10215 }
10216 
10217 void Assembler::decl(Register dst) {
10218   // Don't use it directly. Use MacroAssembler::decrementl() instead.
10219   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
10220   int encode = prefix_and_encode(dst->encoding());
10221   emit_int16((unsigned char)0xFF, (0xC8 | encode));
10222 }
10223 
10224 void Assembler::decq(Register dst) {
10225   // Don't use it directly. Use MacroAssembler::decrementq() instead.
10226   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10227   int encode = prefixq_and_encode(dst->encoding());
10228   emit_int16((unsigned char)0xFF, 0xC8 | encode);
10229 }
10230 
10231 void Assembler::decq(Address dst) {
10232   // Don't use it directly. Use MacroAssembler::decrementq() instead.
10233   InstructionMark im(this);
10234   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
10235   emit_operand(rcx, dst);
10236 }
10237 
10238 void Assembler::fxrstor(Address src) {
10239   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
10240   emit_operand(as_Register(1), src);
10241 }
10242 
10243 void Assembler::xrstor(Address src) {
10244   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
10245   emit_operand(as_Register(5), src);
10246 }
10247 
10248 void Assembler::fxsave(Address dst) {
10249   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
10250   emit_operand(as_Register(0), dst);
10251 }
10252 
10253 void Assembler::xsave(Address dst) {
10254   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
10255   emit_operand(as_Register(4), dst);
10256 }
10257 
10258 void Assembler::idivq(Register src) {
10259   int encode = prefixq_and_encode(src->encoding());
10260   emit_int16((unsigned char)0xF7, (0xF8 | encode));
10261 }
10262 
10263 void Assembler::imulq(Register dst, Register src) {
10264   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10265   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
10266 }
10267 
10268 void Assembler::imulq(Register src) {
10269   int encode = prefixq_and_encode(src->encoding());
10270   emit_int16((unsigned char)0xF7, (0xE8 | encode));
10271 }
10272 
10273 void Assembler::imulq(Register dst, Address src, int32_t value) {
10274   InstructionMark im(this);
10275   prefixq(src, dst);
10276   if (is8bit(value)) {
10277     emit_int8((unsigned char)0x6B);
10278     emit_operand(dst, src);
10279     emit_int8(value);
10280   } else {
10281     emit_int8((unsigned char)0x69);
10282     emit_operand(dst, src);
10283     emit_int32(value);
10284   }
10285 }
10286 
10287 void Assembler::imulq(Register dst, Register src, int value) {
10288   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10289   if (is8bit(value)) {
10290     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
10291   } else {
10292     emit_int16(0x69, (0xC0 | encode));
10293     emit_int32(value);
10294   }
10295 }
10296 
10297 void Assembler::imulq(Register dst, Address src) {
10298   InstructionMark im(this);
10299   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
10300   emit_operand(dst, src);
10301 }
10302 
10303 void Assembler::incl(Register dst) {
10304   // Don't use it directly. Use MacroAssembler::incrementl() instead.
10305   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10306   int encode = prefix_and_encode(dst->encoding());
10307   emit_int16((unsigned char)0xFF, (0xC0 | encode));
10308 }
10309 
10310 void Assembler::incq(Register dst) {
10311   // Don't use it directly. Use MacroAssembler::incrementq() instead.
10312   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
10313   int encode = prefixq_and_encode(dst->encoding());
10314   emit_int16((unsigned char)0xFF, (0xC0 | encode));
10315 }
10316 
10317 void Assembler::incq(Address dst) {
10318   // Don't use it directly. Use MacroAssembler::incrementq() instead.
10319   InstructionMark im(this);
10320   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
10321   emit_operand(rax, dst);
10322 }
10323 
10324 void Assembler::lea(Register dst, Address src) {
10325   leaq(dst, src);
10326 }
10327 
10328 void Assembler::leaq(Register dst, Address src) {
10329   InstructionMark im(this);
10330   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
10331   emit_operand(dst, src);
10332 }
10333 
10334 void Assembler::mov64(Register dst, int64_t imm64) {
10335   InstructionMark im(this);
10336   int encode = prefixq_and_encode(dst->encoding());
10337   emit_int8(0xB8 | encode);
10338   emit_int64(imm64);
10339 }
10340 
10341 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
10342   InstructionMark im(this);
10343   int encode = prefixq_and_encode(dst->encoding());
10344   emit_int8(0xB8 | encode);
10345   emit_data64(imm64, rtype, format);
10346 }
10347 
10348 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
10349   InstructionMark im(this);
10350   int encode = prefixq_and_encode(dst->encoding());
10351   emit_int8(0xB8 | encode);
10352   emit_data64(imm64, rspec);
10353 }
10354 
10355 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
10356   InstructionMark im(this);
10357   int encode = prefix_and_encode(dst->encoding());
10358   emit_int8(0xB8 | encode);
10359   emit_data((int)imm32, rspec, narrow_oop_operand);
10360 }
10361 
10362 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
10363   InstructionMark im(this);
10364   prefix(dst);
10365   emit_int8((unsigned char)0xC7);
10366   emit_operand(rax, dst, 4);
10367   emit_data((int)imm32, rspec, narrow_oop_operand);
10368 }
10369 
10370 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10371   InstructionMark im(this);
10372   int encode = prefix_and_encode(src1->encoding());
10373   emit_int16((unsigned char)0x81, (0xF8 | encode));
10374   emit_data((int)imm32, rspec, narrow_oop_operand);
10375 }
10376 
10377 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10378   InstructionMark im(this);
10379   prefix(src1);
10380   emit_int8((unsigned char)0x81);
10381   emit_operand(rax, src1, 4);
10382   emit_data((int)imm32, rspec, narrow_oop_operand);
10383 }
10384 
10385 void Assembler::lzcntq(Register dst, Register src) {
10386   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
10387   emit_int8((unsigned char)0xF3);
10388   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10389   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
10390 }
10391 
10392 void Assembler::movdq(XMMRegister dst, Register src) {
10393   // table D-1 says MMX/SSE2
10394   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10395   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10396   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10397   emit_int16(0x6E, (0xC0 | encode));
10398 }
10399 
10400 void Assembler::movdq(Register dst, XMMRegister src) {
10401   // table D-1 says MMX/SSE2
10402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
10403   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10404   // swap src/dst to get correct prefix
10405   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10406   emit_int16(0x7E,
10407              (0xC0 | encode));
10408 }
10409 
10410 void Assembler::movq(Register dst, Register src) {
10411   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10412   emit_int16((unsigned char)0x8B,
10413              (0xC0 | encode));
10414 }
10415 
10416 void Assembler::movq(Register dst, Address src) {
10417   InstructionMark im(this);
10418   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
10419   emit_operand(dst, src);
10420 }
10421 
10422 void Assembler::movq(Address dst, Register src) {
10423   InstructionMark im(this);
10424   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
10425   emit_operand(src, dst);
10426 }
10427 
10428 void Assembler::movq(Address dst, int32_t imm32) {
10429   InstructionMark im(this);
10430   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
10431   emit_operand(as_Register(0), dst);
10432   emit_int32(imm32);
10433 }
10434 
10435 void Assembler::movq(Register dst, int32_t imm32) {
10436   int encode = prefixq_and_encode(dst->encoding());
10437   emit_int16((unsigned char)0xC7, (0xC0 | encode));
10438   emit_int32(imm32);
10439 }
10440 
10441 void Assembler::movsbq(Register dst, Address src) {
10442   InstructionMark im(this);
10443   emit_int24(get_prefixq(src, dst),
10444              0x0F,
10445              (unsigned char)0xBE);
10446   emit_operand(dst, src);
10447 }
10448 
10449 void Assembler::movsbq(Register dst, Register src) {
10450   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10451   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
10452 }
10453 
10454 void Assembler::movslq(Register dst, int32_t imm32) {
10455   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
10456   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
10457   // as a result we shouldn't use until tested at runtime...
10458   ShouldNotReachHere();
10459   InstructionMark im(this);
10460   int encode = prefixq_and_encode(dst->encoding());
10461   emit_int8(0xC7 | encode);
10462   emit_int32(imm32);
10463 }
10464 
10465 void Assembler::movslq(Address dst, int32_t imm32) {
10466   assert(is_simm32(imm32), "lost bits");
10467   InstructionMark im(this);
10468   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
10469   emit_operand(rax, dst, 4);
10470   emit_int32(imm32);
10471 }
10472 
10473 void Assembler::movslq(Register dst, Address src) {
10474   InstructionMark im(this);
10475   emit_int16(get_prefixq(src, dst), 0x63);
10476   emit_operand(dst, src);
10477 }
10478 
10479 void Assembler::movslq(Register dst, Register src) {
10480   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10481   emit_int16(0x63, (0xC0 | encode));
10482 }
10483 
10484 void Assembler::movswq(Register dst, Address src) {
10485   InstructionMark im(this);
10486   emit_int24(get_prefixq(src, dst),
10487              0x0F,
10488              (unsigned char)0xBF);
10489   emit_operand(dst, src);
10490 }
10491 
10492 void Assembler::movswq(Register dst, Register src) {
10493   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10494   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
10495 }
10496 
10497 void Assembler::movzbq(Register dst, Address src) {
10498   InstructionMark im(this);
10499   emit_int24(get_prefixq(src, dst),
10500              0x0F,
10501              (unsigned char)0xB6);
10502   emit_operand(dst, src);
10503 }
10504 
10505 void Assembler::movzbq(Register dst, Register src) {
10506   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10507   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
10508 }
10509 
10510 void Assembler::movzwq(Register dst, Address src) {
10511   InstructionMark im(this);
10512   emit_int24(get_prefixq(src, dst),
10513              0x0F,
10514              (unsigned char)0xB7);
10515   emit_operand(dst, src);
10516 }
10517 
10518 void Assembler::movzwq(Register dst, Register src) {
10519   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10520   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
10521 }
10522 
10523 void Assembler::mulq(Address src) {
10524   InstructionMark im(this);
10525   emit_int16(get_prefixq(src), (unsigned char)0xF7);
10526   emit_operand(rsp, src);
10527 }
10528 
10529 void Assembler::mulq(Register src) {
10530   int encode = prefixq_and_encode(src->encoding());
10531   emit_int16((unsigned char)0xF7, (0xE0 | encode));
10532 }
10533 
10534 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
10535   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
10536   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10537   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
10538   emit_int16((unsigned char)0xF6, (0xC0 | encode));
10539 }
10540 
10541 void Assembler::negq(Register dst) {
10542   int encode = prefixq_and_encode(dst->encoding());
10543   emit_int16((unsigned char)0xF7, (0xD8 | encode));
10544 }
10545 
10546 void Assembler::negq(Address dst) {
10547   InstructionMark im(this);
10548   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
10549   emit_operand(as_Register(3), dst);
10550 }
10551 
10552 void Assembler::notq(Register dst) {
10553   int encode = prefixq_and_encode(dst->encoding());
10554   emit_int16((unsigned char)0xF7, (0xD0 | encode));
10555 }
10556 
10557 void Assembler::btsq(Address dst, int imm8) {
10558   assert(isByte(imm8), "not a byte");
10559   InstructionMark im(this);
10560   emit_int24(get_prefixq(dst),
10561              0x0F,
10562              (unsigned char)0xBA);
10563   emit_operand(rbp /* 5 */, dst, 1);
10564   emit_int8(imm8);
10565 }
10566 
10567 void Assembler::btrq(Address dst, int imm8) {
10568   assert(isByte(imm8), "not a byte");
10569   InstructionMark im(this);
10570   emit_int24(get_prefixq(dst),
10571              0x0F,
10572              (unsigned char)0xBA);
10573   emit_operand(rsi /* 6 */, dst, 1);
10574   emit_int8(imm8);
10575 }
10576 
10577 void Assembler::orq(Address dst, int32_t imm32) {
10578   InstructionMark im(this);
10579   prefixq(dst);
10580   emit_arith_operand(0x81, as_Register(1), dst, imm32);
10581 }
10582 
10583 void Assembler::orq(Address dst, Register src) {
10584   InstructionMark im(this);
10585   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
10586   emit_operand(src, dst);
10587 }
10588 
10589 void Assembler::orq(Register dst, int32_t imm32) {
10590   (void) prefixq_and_encode(dst->encoding());
10591   emit_arith(0x81, 0xC8, dst, imm32);
10592 }
10593 
10594 void Assembler::orq(Register dst, Address src) {
10595   InstructionMark im(this);
10596   emit_int16(get_prefixq(src, dst), 0x0B);
10597   emit_operand(dst, src);
10598 }
10599 
10600 void Assembler::orq(Register dst, Register src) {
10601   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10602   emit_arith(0x0B, 0xC0, dst, src);
10603 }
10604 
10605 void Assembler::popcntq(Register dst, Address src) {
10606   assert(VM_Version::supports_popcnt(), "must support");
10607   InstructionMark im(this);
10608   emit_int32((unsigned char)0xF3,
10609              get_prefixq(src, dst),
10610              0x0F,
10611              (unsigned char)0xB8);
10612   emit_operand(dst, src);
10613 }
10614 
10615 void Assembler::popcntq(Register dst, Register src) {
10616   assert(VM_Version::supports_popcnt(), "must support");
10617   emit_int8((unsigned char)0xF3);
10618   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
10619   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
10620 }
10621 
10622 void Assembler::popq(Address dst) {
10623   InstructionMark im(this);
10624   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
10625   emit_operand(rax, dst);
10626 }
10627 
10628 void Assembler::popq(Register dst) {
10629   emit_int8((unsigned char)0x58 | dst->encoding());
10630 }
10631 
10632 // Precomputable: popa, pusha, vzeroupper
10633 
10634 // The result of these routines are invariant from one invocation to another
10635 // invocation for the duration of a run. Caching the result on bootstrap
10636 // and copying it out on subsequent invocations can thus be beneficial
10637 static bool     precomputed = false;
10638 
10639 static u_char* popa_code  = NULL;
10640 static int     popa_len   = 0;
10641 
10642 static u_char* pusha_code = NULL;
10643 static int     pusha_len  = 0;
10644 
10645 static u_char* vzup_code  = NULL;
10646 static int     vzup_len   = 0;
10647 
10648 void Assembler::precompute_instructions() {
10649   assert(!Universe::is_fully_initialized(), "must still be single threaded");
10650   guarantee(!precomputed, "only once");
10651   precomputed = true;
10652   ResourceMark rm;
10653 
10654   // Make a temporary buffer big enough for the routines we're capturing
10655   int size = 256;
10656   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
10657   CodeBuffer buffer((address)tmp_code, size);
10658   MacroAssembler masm(&buffer);
10659 
10660   address begin_popa  = masm.code_section()->end();
10661   masm.popa_uncached();
10662   address end_popa    = masm.code_section()->end();
10663   masm.pusha_uncached();
10664   address end_pusha   = masm.code_section()->end();
10665   masm.vzeroupper_uncached();
10666   address end_vzup    = masm.code_section()->end();
10667 
10668   // Save the instructions to permanent buffers.
10669   popa_len = (int)(end_popa - begin_popa);
10670   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
10671   memcpy(popa_code, begin_popa, popa_len);
10672 
10673   pusha_len = (int)(end_pusha - end_popa);
10674   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
10675   memcpy(pusha_code, end_popa, pusha_len);
10676 
10677   vzup_len = (int)(end_vzup - end_pusha);
10678   if (vzup_len > 0) {
10679     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
10680     memcpy(vzup_code, end_pusha, vzup_len);
10681   } else {
10682     vzup_code = pusha_code; // dummy
10683   }
10684 
10685   assert(masm.code()->total_oop_size() == 0 &&
10686          masm.code()->total_metadata_size() == 0 &&
10687          masm.code()->total_relocation_size() == 0,
10688          "pre-computed code can't reference oops, metadata or contain relocations");
10689 }
10690 
10691 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
10692   assert(src != NULL, "code to copy must have been pre-computed");
10693   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
10694   address end = code_section->end();
10695   memcpy(end, src, src_len);
10696   code_section->set_end(end + src_len);
10697 }
10698 
10699 void Assembler::popa() { // 64bit
10700   emit_copy(code_section(), popa_code, popa_len);
10701 }
10702 
10703 void Assembler::popa_uncached() { // 64bit
10704   movq(r15, Address(rsp, 0));
10705   movq(r14, Address(rsp, wordSize));
10706   movq(r13, Address(rsp, 2 * wordSize));
10707   movq(r12, Address(rsp, 3 * wordSize));
10708   movq(r11, Address(rsp, 4 * wordSize));
10709   movq(r10, Address(rsp, 5 * wordSize));
10710   movq(r9,  Address(rsp, 6 * wordSize));
10711   movq(r8,  Address(rsp, 7 * wordSize));
10712   movq(rdi, Address(rsp, 8 * wordSize));
10713   movq(rsi, Address(rsp, 9 * wordSize));
10714   movq(rbp, Address(rsp, 10 * wordSize));
10715   // Skip rsp as it is restored automatically to the value
10716   // before the corresponding pusha when popa is done.
10717   movq(rbx, Address(rsp, 12 * wordSize));
10718   movq(rdx, Address(rsp, 13 * wordSize));
10719   movq(rcx, Address(rsp, 14 * wordSize));
10720   movq(rax, Address(rsp, 15 * wordSize));
10721 
10722   addq(rsp, 16 * wordSize);
10723 }
10724 
10725 // Does not actually store the value of rsp on the stack.
10726 // The slot for rsp just contains an arbitrary value.
10727 void Assembler::pusha() { // 64bit
10728   emit_copy(code_section(), pusha_code, pusha_len);
10729 }
10730 
10731 // Does not actually store the value of rsp on the stack.
10732 // The slot for rsp just contains an arbitrary value.
10733 void Assembler::pusha_uncached() { // 64bit
10734   subq(rsp, 16 * wordSize);
10735 
10736   movq(Address(rsp, 15 * wordSize), rax);
10737   movq(Address(rsp, 14 * wordSize), rcx);
10738   movq(Address(rsp, 13 * wordSize), rdx);
10739   movq(Address(rsp, 12 * wordSize), rbx);
10740   // Skip rsp as the value is normally not used. There are a few places where
10741   // the original value of rsp needs to be known but that can be computed
10742   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
10743   movq(Address(rsp, 10 * wordSize), rbp);
10744   movq(Address(rsp, 9 * wordSize), rsi);
10745   movq(Address(rsp, 8 * wordSize), rdi);
10746   movq(Address(rsp, 7 * wordSize), r8);
10747   movq(Address(rsp, 6 * wordSize), r9);
10748   movq(Address(rsp, 5 * wordSize), r10);
10749   movq(Address(rsp, 4 * wordSize), r11);
10750   movq(Address(rsp, 3 * wordSize), r12);
10751   movq(Address(rsp, 2 * wordSize), r13);
10752   movq(Address(rsp, wordSize), r14);
10753   movq(Address(rsp, 0), r15);
10754 }
10755 
10756 void Assembler::vzeroupper() {
10757   emit_copy(code_section(), vzup_code, vzup_len);
10758 }
10759 
10760 void Assembler::pushq(Address src) {
10761   InstructionMark im(this);
10762   emit_int16(get_prefixq(src), (unsigned char)0xFF);
10763   emit_operand(rsi, src);
10764 }
10765 
10766 void Assembler::rclq(Register dst, int imm8) {
10767   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10768   int encode = prefixq_and_encode(dst->encoding());
10769   if (imm8 == 1) {
10770     emit_int16((unsigned char)0xD1, (0xD0 | encode));
10771   } else {
10772     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
10773   }
10774 }
10775 
10776 void Assembler::rcrq(Register dst, int imm8) {
10777   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10778   int encode = prefixq_and_encode(dst->encoding());
10779   if (imm8 == 1) {
10780     emit_int16((unsigned char)0xD1, (0xD8 | encode));
10781   } else {
10782     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
10783   }
10784 }
10785 
10786 
10787 void Assembler::rorxq(Register dst, Register src, int imm8) {
10788   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
10789   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10790   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
10791   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
10792 }
10793 
10794 void Assembler::rorxd(Register dst, Register src, int imm8) {
10795   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
10796   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10797   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
10798   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
10799 }
10800 
10801 #ifdef _LP64
10802 void Assembler::salq(Address dst, int imm8) {
10803   InstructionMark im(this);
10804   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10805   if (imm8 == 1) {
10806     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
10807     emit_operand(as_Register(4), dst);
10808   }
10809   else {
10810     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
10811     emit_operand(as_Register(4), dst);
10812     emit_int8(imm8);
10813   }
10814 }
10815 
10816 void Assembler::salq(Address dst) {
10817   InstructionMark im(this);
10818   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
10819   emit_operand(as_Register(4), dst);
10820 }
10821 
10822 void Assembler::salq(Register dst, int imm8) {
10823   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10824   int encode = prefixq_and_encode(dst->encoding());
10825   if (imm8 == 1) {
10826     emit_int16((unsigned char)0xD1, (0xE0 | encode));
10827   } else {
10828     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
10829   }
10830 }
10831 
10832 void Assembler::salq(Register dst) {
10833   int encode = prefixq_and_encode(dst->encoding());
10834   emit_int16((unsigned char)0xD3, (0xE0 | encode));
10835 }
10836 
10837 void Assembler::sarq(Address dst, int imm8) {
10838   InstructionMark im(this);
10839   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10840   if (imm8 == 1) {
10841     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
10842     emit_operand(as_Register(7), dst);
10843   }
10844   else {
10845     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
10846     emit_operand(as_Register(7), dst);
10847     emit_int8(imm8);
10848   }
10849 }
10850 
10851 void Assembler::sarq(Address dst) {
10852   InstructionMark im(this);
10853   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
10854   emit_operand(as_Register(7), dst);
10855 }
10856 
10857 void Assembler::sarq(Register dst, int imm8) {
10858   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10859   int encode = prefixq_and_encode(dst->encoding());
10860   if (imm8 == 1) {
10861     emit_int16((unsigned char)0xD1, (0xF8 | encode));
10862   } else {
10863     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
10864   }
10865 }
10866 
10867 void Assembler::sarq(Register dst) {
10868   int encode = prefixq_and_encode(dst->encoding());
10869   emit_int16((unsigned char)0xD3, (0xF8 | encode));
10870 }
10871 #endif
10872 
10873 void Assembler::sbbq(Address dst, int32_t imm32) {
10874   InstructionMark im(this);
10875   prefixq(dst);
10876   emit_arith_operand(0x81, rbx, dst, imm32);
10877 }
10878 
10879 void Assembler::sbbq(Register dst, int32_t imm32) {
10880   (void) prefixq_and_encode(dst->encoding());
10881   emit_arith(0x81, 0xD8, dst, imm32);
10882 }
10883 
10884 void Assembler::sbbq(Register dst, Address src) {
10885   InstructionMark im(this);
10886   emit_int16(get_prefixq(src, dst), 0x1B);
10887   emit_operand(dst, src);
10888 }
10889 
10890 void Assembler::sbbq(Register dst, Register src) {
10891   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10892   emit_arith(0x1B, 0xC0, dst, src);
10893 }
10894 
10895 void Assembler::shlq(Register dst, int imm8) {
10896   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10897   int encode = prefixq_and_encode(dst->encoding());
10898   if (imm8 == 1) {
10899     emit_int16((unsigned char)0xD1, (0xE0 | encode));
10900   } else {
10901     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
10902   }
10903 }
10904 
10905 void Assembler::shlq(Register dst) {
10906   int encode = prefixq_and_encode(dst->encoding());
10907   emit_int16((unsigned char)0xD3, (0xE0 | encode));
10908 }
10909 
10910 void Assembler::shrq(Register dst, int imm8) {
10911   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10912   int encode = prefixq_and_encode(dst->encoding());
10913   if (imm8 == 1) {
10914     emit_int16((unsigned char)0xD1, (0xE8 | encode));
10915   }
10916   else {
10917     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
10918   }
10919 }
10920 
10921 void Assembler::shrq(Register dst) {
10922   int encode = prefixq_and_encode(dst->encoding());
10923   emit_int16((unsigned char)0xD3, 0xE8 | encode);
10924 }
10925 
10926 void Assembler::shrq(Address dst) {
10927   InstructionMark im(this);
10928   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
10929   emit_operand(as_Register(5), dst);
10930 }
10931 
10932 void Assembler::shrq(Address dst, int imm8) {
10933   InstructionMark im(this);
10934   assert(isShiftCount(imm8 >> 1), "illegal shift count");
10935   if (imm8 == 1) {
10936     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
10937     emit_operand(as_Register(5), dst);
10938   }
10939   else {
10940     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
10941     emit_operand(as_Register(5), dst);
10942     emit_int8(imm8);
10943   }
10944 }
10945 
10946 void Assembler::subq(Address dst, int32_t imm32) {
10947   InstructionMark im(this);
10948   prefixq(dst);
10949   emit_arith_operand(0x81, rbp, dst, imm32);
10950 }
10951 
10952 void Assembler::subq(Address dst, Register src) {
10953   InstructionMark im(this);
10954   emit_int16(get_prefixq(dst, src), 0x29);
10955   emit_operand(src, dst);
10956 }
10957 
10958 void Assembler::subq(Register dst, int32_t imm32) {
10959   (void) prefixq_and_encode(dst->encoding());
10960   emit_arith(0x81, 0xE8, dst, imm32);
10961 }
10962 
10963 // Force generation of a 4 byte immediate value even if it fits into 8bit
10964 void Assembler::subq_imm32(Register dst, int32_t imm32) {
10965   (void) prefixq_and_encode(dst->encoding());
10966   emit_arith_imm32(0x81, 0xE8, dst, imm32);
10967 }
10968 
10969 void Assembler::subq(Register dst, Address src) {
10970   InstructionMark im(this);
10971   emit_int16(get_prefixq(src, dst), 0x2B);
10972   emit_operand(dst, src);
10973 }
10974 
10975 void Assembler::subq(Register dst, Register src) {
10976   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10977   emit_arith(0x2B, 0xC0, dst, src);
10978 }
10979 
10980 void Assembler::testq(Address dst, int32_t imm32) {
10981   InstructionMark im(this);
10982   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
10983   emit_operand(as_Register(0), dst);
10984   emit_int32(imm32);
10985 }
10986 
10987 void Assembler::testq(Register dst, int32_t imm32) {
10988   // not using emit_arith because test
10989   // doesn't support sign-extension of
10990   // 8bit operands
10991   int encode = dst->encoding();
10992   encode = prefixq_and_encode(encode);
10993   emit_int16((unsigned char)0xF7, (0xC0 | encode));
10994   emit_int32(imm32);
10995 }
10996 
10997 void Assembler::testq(Register dst, Register src) {
10998   (void) prefixq_and_encode(dst->encoding(), src->encoding());
10999   emit_arith(0x85, 0xC0, dst, src);
11000 }
11001 
11002 void Assembler::testq(Register dst, Address src) {
11003   InstructionMark im(this);
11004   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
11005   emit_operand(dst, src);
11006 }
11007 
11008 void Assembler::xaddq(Address dst, Register src) {
11009   InstructionMark im(this);
11010   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
11011   emit_operand(src, dst);
11012 }
11013 
11014 void Assembler::xchgq(Register dst, Address src) {
11015   InstructionMark im(this);
11016   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
11017   emit_operand(dst, src);
11018 }
11019 
11020 void Assembler::xchgq(Register dst, Register src) {
11021   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
11022   emit_int16((unsigned char)0x87, (0xc0 | encode));
11023 }
11024 
11025 void Assembler::xorq(Register dst, Register src) {
11026   (void) prefixq_and_encode(dst->encoding(), src->encoding());
11027   emit_arith(0x33, 0xC0, dst, src);
11028 }
11029 
11030 void Assembler::xorq(Register dst, Address src) {
11031   InstructionMark im(this);
11032   emit_int16(get_prefixq(src, dst), 0x33);
11033   emit_operand(dst, src);
11034 }
11035 
11036 void Assembler::xorq(Register dst, int32_t imm32) {
11037   (void) prefixq_and_encode(dst->encoding());
11038   emit_arith(0x81, 0xF0, dst, imm32);
11039 }
11040 
11041 void Assembler::xorq(Address dst, int32_t imm32) {
11042   InstructionMark im(this);
11043   prefixq(dst);
11044   emit_arith_operand(0x81, as_Register(6), dst, imm32);
11045 }
11046 
11047 void Assembler::xorq(Address dst, Register src) {
11048   InstructionMark im(this);
11049   emit_int16(get_prefixq(dst, src), 0x31);
11050   emit_operand(src, dst);
11051 }
11052 
11053 #endif // !LP64
11054 
11055 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
11056   if (VM_Version::supports_evex()) {
11057     _tuple_type = tuple_type;
11058     _input_size_in_bits = input_size_in_bits;
11059   }
11060 }