1 /*
    2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "gc/shared/cardTableBarrierSet.hpp"
   29 #include "interpreter/interpreter.hpp"
   30 #include "memory/resourceArea.hpp"
   31 #include "memory/universe.hpp"
   32 #include "prims/methodHandles.hpp"
   33 #include "runtime/objectMonitor.hpp"
   34 #include "runtime/os.hpp"
   35 #include "runtime/sharedRuntime.hpp"
   36 #include "runtime/stubRoutines.hpp"
   37 #include "runtime/vm_version.hpp"
   38 #include "utilities/checkedCast.hpp"
   39 #include "utilities/macros.hpp"
   40 
   41 #ifdef PRODUCT
   42 #define BLOCK_COMMENT(str) /* nothing */
   43 #define STOP(error) stop(error)
   44 #else
   45 #define BLOCK_COMMENT(str) block_comment(str)
   46 #define STOP(error) block_comment(error); stop(error)
   47 #endif
   48 
   49 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   50 // Implementation of AddressLiteral
   51 
   52 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
   53 static const unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
   54   // -----------------Table 4.5 -------------------- //
   55   16, 32, 64,  // EVEX_FV(0)
   56   4,  4,  4,   // EVEX_FV(1) - with Evex.b
   57   16, 32, 64,  // EVEX_FV(2) - with Evex.w
   58   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
   59   8,  16, 32,  // EVEX_HV(0)
   60   4,  4,  4,   // EVEX_HV(1) - with Evex.b
   61   // -----------------Table 4.6 -------------------- //
   62   16, 32, 64,  // EVEX_FVM(0)
   63   1,  1,  1,   // EVEX_T1S(0)
   64   2,  2,  2,   // EVEX_T1S(1)
   65   4,  4,  4,   // EVEX_T1S(2)
   66   8,  8,  8,   // EVEX_T1S(3)
   67   4,  4,  4,   // EVEX_T1F(0)
   68   8,  8,  8,   // EVEX_T1F(1)
   69   8,  8,  8,   // EVEX_T2(0)
   70   0,  16, 16,  // EVEX_T2(1)
   71   0,  16, 16,  // EVEX_T4(0)
   72   0,  0,  32,  // EVEX_T4(1)
   73   0,  0,  32,  // EVEX_T8(0)
   74   8,  16, 32,  // EVEX_HVM(0)
   75   4,  8,  16,  // EVEX_QVM(0)
   76   2,  4,  8,   // EVEX_OVM(0)
   77   16, 16, 16,  // EVEX_M128(0)
   78   8,  32, 64,  // EVEX_DUP(0)
   79   0,  0,  0    // EVEX_NTUP
   80 };
   81 
   82 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
   83   _is_lval = false;
   84   _target = target;
   85   switch (rtype) {
   86   case relocInfo::oop_type:
   87   case relocInfo::metadata_type:
   88     // Oops are a special case. Normally they would be their own section
   89     // but in cases like icBuffer they are literals in the code stream that
   90     // we don't have a section for. We use none so that we get a literal address
   91     // which is always patchable.
   92     break;
   93   case relocInfo::external_word_type:
   94     _rspec = external_word_Relocation::spec(target);
   95     break;
   96   case relocInfo::internal_word_type:
   97     _rspec = internal_word_Relocation::spec(target);
   98     break;
   99   case relocInfo::opt_virtual_call_type:
  100     _rspec = opt_virtual_call_Relocation::spec();
  101     break;
  102   case relocInfo::static_call_type:
  103     _rspec = static_call_Relocation::spec();
  104     break;
  105   case relocInfo::runtime_call_type:
  106     _rspec = runtime_call_Relocation::spec();
  107     break;
  108   case relocInfo::poll_type:
  109   case relocInfo::poll_return_type:
  110     _rspec = Relocation::spec_simple(rtype);
  111     break;
  112   case relocInfo::none:
  113     break;
  114   default:
  115     ShouldNotReachHere();
  116     break;
  117   }
  118 }
  119 
  120 // Implementation of Address
  121 
  122 #ifdef _LP64
  123 
  124 Address Address::make_array(ArrayAddress adr) {
  125   // Not implementable on 64bit machines
  126   // Should have been handled higher up the call chain.
  127   ShouldNotReachHere();
  128   return Address();
  129 }
  130 
  131 // exceedingly dangerous constructor
  132 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  133   _base  = noreg;
  134   _index = noreg;
  135   _scale = no_scale;
  136   _disp  = disp;
  137   _xmmindex = xnoreg;
  138   _isxmmindex = false;
  139   switch (rtype) {
  140     case relocInfo::external_word_type:
  141       _rspec = external_word_Relocation::spec(loc);
  142       break;
  143     case relocInfo::internal_word_type:
  144       _rspec = internal_word_Relocation::spec(loc);
  145       break;
  146     case relocInfo::runtime_call_type:
  147       // HMM
  148       _rspec = runtime_call_Relocation::spec();
  149       break;
  150     case relocInfo::poll_type:
  151     case relocInfo::poll_return_type:
  152       _rspec = Relocation::spec_simple(rtype);
  153       break;
  154     case relocInfo::none:
  155       break;
  156     default:
  157       ShouldNotReachHere();
  158   }
  159 }
  160 #else // LP64
  161 
  162 Address Address::make_array(ArrayAddress adr) {
  163   AddressLiteral base = adr.base();
  164   Address index = adr.index();
  165   assert(index._disp == 0, "must not have disp"); // maybe it can?
  166   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
  167   array._rspec = base._rspec;
  168   return array;
  169 }
  170 
  171 // exceedingly dangerous constructor
  172 Address::Address(address loc, RelocationHolder spec) {
  173   _base  = noreg;
  174   _index = noreg;
  175   _scale = no_scale;
  176   _disp  = (intptr_t) loc;
  177   _rspec = spec;
  178   _xmmindex = xnoreg;
  179   _isxmmindex = false;
  180 }
  181 
  182 #endif // _LP64
  183 
  184 
  185 
  186 // Convert the raw encoding form into the form expected by the constructor for
  187 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
  188 // that to noreg for the Address constructor.
  189 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
  190   RelocationHolder rspec = RelocationHolder::none;
  191   if (disp_reloc != relocInfo::none) {
  192     rspec = Relocation::spec_simple(disp_reloc);
  193   }
  194   bool valid_index = index != rsp->encoding();
  195   if (valid_index) {
  196     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
  197     madr._rspec = rspec;
  198     return madr;
  199   } else {
  200     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
  201     madr._rspec = rspec;
  202     return madr;
  203   }
  204 }
  205 
  206 // Implementation of Assembler
  207 
  208 int AbstractAssembler::code_fill_byte() {
  209   return (u_char)'\xF4'; // hlt
  210 }
  211 
  212 void Assembler::init_attributes(void) {
  213   _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
  214   _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
  215   _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
  216   _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
  217   NOT_LP64(_is_managed = false;)
  218   _attributes = nullptr;
  219 }
  220 
  221 void Assembler::set_attributes(InstructionAttr* attributes) {
  222   // Record the assembler in the attributes, so the attributes destructor can
  223   // clear the assembler's attributes, cleaning up the otherwise dangling
  224   // pointer.  gcc13 has a false positive warning, because it doesn't tie that
  225   // cleanup to the assignment of _attributes here.
  226   attributes->set_current_assembler(this);
  227   PRAGMA_DIAG_PUSH
  228   PRAGMA_DANGLING_POINTER_IGNORED
  229   _attributes = attributes;
  230   PRAGMA_DIAG_POP
  231 }
  232 
  233 void Assembler::membar(Membar_mask_bits order_constraint) {
  234   // We only have to handle StoreLoad
  235   if (order_constraint & StoreLoad) {
  236     // All usable chips support "locked" instructions which suffice
  237     // as barriers, and are much faster than the alternative of
  238     // using cpuid instruction. We use here a locked add [esp-C],0.
  239     // This is conveniently otherwise a no-op except for blowing
  240     // flags, and introducing a false dependency on target memory
  241     // location. We can't do anything with flags, but we can avoid
  242     // memory dependencies in the current method by locked-adding
  243     // somewhere else on the stack. Doing [esp+C] will collide with
  244     // something on stack in current method, hence we go for [esp-C].
  245     // It is convenient since it is almost always in data cache, for
  246     // any small C.  We need to step back from SP to avoid data
  247     // dependencies with other things on below SP (callee-saves, for
  248     // example). Without a clear way to figure out the minimal safe
  249     // distance from SP, it makes sense to step back the complete
  250     // cache line, as this will also avoid possible second-order effects
  251     // with locked ops against the cache line. Our choice of offset
  252     // is bounded by x86 operand encoding, which should stay within
  253     // [-128; +127] to have the 8-byte displacement encoding.
  254     //
  255     // Any change to this code may need to revisit other places in
  256     // the code where this idiom is used, in particular the
  257     // orderAccess code.
  258 
  259     int offset = -VM_Version::L1_line_size();
  260     if (offset < -128) {
  261       offset = -128;
  262     }
  263 
  264     lock();
  265     addl(Address(rsp, offset), 0);// Assert the lock# signal here
  266   }
  267 }
  268 
  269 // make this go away someday
  270 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
  271   if (rtype == relocInfo::none)
  272     emit_int32(data);
  273   else
  274     emit_data(data, Relocation::spec_simple(rtype), format);
  275 }
  276 
  277 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
  278   assert(imm_operand == 0, "default format must be immediate in this file");
  279   assert(inst_mark() != nullptr, "must be inside InstructionMark");
  280   if (rspec.type() !=  relocInfo::none) {
  281     #ifdef ASSERT
  282       check_relocation(rspec, format);
  283     #endif
  284     // Do not use AbstractAssembler::relocate, which is not intended for
  285     // embedded words.  Instead, relocate to the enclosing instruction.
  286 
  287     // hack. call32 is too wide for mask so use disp32
  288     if (format == call32_operand)
  289       code_section()->relocate(inst_mark(), rspec, disp32_operand);
  290     else
  291       code_section()->relocate(inst_mark(), rspec, format);
  292   }
  293   emit_int32(data);
  294 }
  295 
  296 static int encode(Register r) {
  297   int enc = r->encoding();
  298   if (enc >= 8) {
  299     enc -= 8;
  300   }
  301   return enc;
  302 }
  303 
  304 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
  305   assert(dst->has_byte_register(), "must have byte register");
  306   assert(isByte(op1) && isByte(op2), "wrong opcode");
  307   assert(isByte(imm8), "not a byte");
  308   assert((op1 & 0x01) == 0, "should be 8bit operation");
  309   emit_int24(op1, (op2 | encode(dst)), imm8);
  310 }
  311 
  312 
  313 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
  314   assert(isByte(op1) && isByte(op2), "wrong opcode");
  315   assert(op1 == 0x81, "Unexpected opcode");
  316   if (is8bit(imm32)) {
  317     emit_int24(op1 | 0x02,        // set sign bit
  318                op2 | encode(dst),
  319                imm32 & 0xFF);
  320   } else if (dst == rax) {
  321     switch (op2) {
  322       case 0xD0: emit_int8(0x15); break; // adc
  323       case 0xC0: emit_int8(0x05); break; // add
  324       case 0xE0: emit_int8(0x25); break; // and
  325       case 0xF8: emit_int8(0x3D); break; // cmp
  326       case 0xC8: emit_int8(0x0D); break; // or
  327       case 0xD8: emit_int8(0x1D); break; // sbb
  328       case 0xE8: emit_int8(0x2D); break; // sub
  329       case 0xF0: emit_int8(0x35); break; // xor
  330       default: ShouldNotReachHere();
  331     }
  332     emit_int32(imm32);
  333   } else {
  334     emit_int16(op1, (op2 | encode(dst)));
  335     emit_int32(imm32);
  336   }
  337 }
  338 
  339 // Force generation of a 4 byte immediate value even if it fits into 8bit
  340 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
  341   assert(isByte(op1) && isByte(op2), "wrong opcode");
  342   assert((op1 & 0x01) == 1, "should be 32bit operation");
  343   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  344   emit_int16(op1, (op2 | encode(dst)));
  345   emit_int32(imm32);
  346 }
  347 
  348 // immediate-to-memory forms
  349 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
  350   assert((op1 & 0x01) == 1, "should be 32bit operation");
  351   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
  352   if (is8bit(imm32)) {
  353     emit_int8(op1 | 0x02); // set sign bit
  354     emit_operand(rm, adr, 1);
  355     emit_int8(imm32 & 0xFF);
  356   } else {
  357     emit_int8(op1);
  358     emit_operand(rm, adr, 4);
  359     emit_int32(imm32);
  360   }
  361 }
  362 
  363 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
  364   assert(op1 == 0x81, "unexpected opcode");
  365   emit_int8(op1);
  366   emit_operand(rm, adr, 4);
  367   emit_int32(imm32);
  368 }
  369 
  370 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
  371   assert(isByte(op1) && isByte(op2), "wrong opcode");
  372   emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
  373 }
  374 
  375 
  376 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
  377                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
  378   int mod_idx = 0;
  379   // We will test if the displacement fits the compressed format and if so
  380   // apply the compression to the displacement iff the result is8bit.
  381   if (VM_Version::supports_evex() && is_evex_inst) {
  382     switch (cur_tuple_type) {
  383     case EVEX_FV:
  384       if ((cur_encoding & VEX_W) == VEX_W) {
  385         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  386       } else {
  387         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  388       }
  389       break;
  390 
  391     case EVEX_HV:
  392       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  393       break;
  394 
  395     case EVEX_FVM:
  396       break;
  397 
  398     case EVEX_T1S:
  399       switch (in_size_in_bits) {
  400       case EVEX_8bit:
  401         break;
  402 
  403       case EVEX_16bit:
  404         mod_idx = 1;
  405         break;
  406 
  407       case EVEX_32bit:
  408         mod_idx = 2;
  409         break;
  410 
  411       case EVEX_64bit:
  412         mod_idx = 3;
  413         break;
  414       }
  415       break;
  416 
  417     case EVEX_T1F:
  418     case EVEX_T2:
  419     case EVEX_T4:
  420       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
  421       break;
  422 
  423     case EVEX_T8:
  424       break;
  425 
  426     case EVEX_HVM:
  427       break;
  428 
  429     case EVEX_QVM:
  430       break;
  431 
  432     case EVEX_OVM:
  433       break;
  434 
  435     case EVEX_M128:
  436       break;
  437 
  438     case EVEX_DUP:
  439       break;
  440 
  441     default:
  442       assert(0, "no valid evex tuple_table entry");
  443       break;
  444     }
  445 
  446     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  447       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
  448       if ((disp % disp_factor) == 0) {
  449         int new_disp = disp / disp_factor;
  450         if ((-0x80 <= new_disp && new_disp < 0x80)) {
  451           disp = new_disp;
  452         }
  453       } else {
  454         return false;
  455       }
  456     }
  457   }
  458   return (-0x80 <= disp && disp < 0x80);
  459 }
  460 
  461 
  462 bool Assembler::emit_compressed_disp_byte(int &disp) {
  463   int mod_idx = 0;
  464   // We will test if the displacement fits the compressed format and if so
  465   // apply the compression to the displacement iff the result is8bit.
  466   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
  467     int evex_encoding = _attributes->get_evex_encoding();
  468     int tuple_type = _attributes->get_tuple_type();
  469     switch (tuple_type) {
  470     case EVEX_FV:
  471       if ((evex_encoding & VEX_W) == VEX_W) {
  472         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
  473       } else {
  474         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  475       }
  476       break;
  477 
  478     case EVEX_HV:
  479       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
  480       break;
  481 
  482     case EVEX_FVM:
  483       break;
  484 
  485     case EVEX_T1S:
  486       switch (_attributes->get_input_size()) {
  487       case EVEX_8bit:
  488         break;
  489 
  490       case EVEX_16bit:
  491         mod_idx = 1;
  492         break;
  493 
  494       case EVEX_32bit:
  495         mod_idx = 2;
  496         break;
  497 
  498       case EVEX_64bit:
  499         mod_idx = 3;
  500         break;
  501       }
  502       break;
  503 
  504     case EVEX_T1F:
  505     case EVEX_T2:
  506     case EVEX_T4:
  507       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
  508       break;
  509 
  510     case EVEX_T8:
  511       break;
  512 
  513     case EVEX_HVM:
  514       break;
  515 
  516     case EVEX_QVM:
  517       break;
  518 
  519     case EVEX_OVM:
  520       break;
  521 
  522     case EVEX_M128:
  523       break;
  524 
  525     case EVEX_DUP:
  526       break;
  527 
  528     default:
  529       assert(0, "no valid evex tuple_table entry");
  530       break;
  531     }
  532 
  533     int vector_len = _attributes->get_vector_len();
  534     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
  535       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
  536       if ((disp % disp_factor) == 0) {
  537         int new_disp = disp / disp_factor;
  538         if (is8bit(new_disp)) {
  539           disp = new_disp;
  540         }
  541       } else {
  542         return false;
  543       }
  544     }
  545   }
  546   return is8bit(disp);
  547 }
  548 
  549 static bool is_valid_encoding(int reg_enc) {
  550   return reg_enc >= 0;
  551 }
  552 
  553 static int raw_encode(Register reg) {
  554   assert(reg == noreg || reg->is_valid(), "sanity");
  555   int reg_enc = reg->raw_encoding();
  556   assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
  557   return reg_enc;
  558 }
  559 
  560 static int raw_encode(XMMRegister xmmreg) {
  561   assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
  562   int xmmreg_enc = xmmreg->raw_encoding();
  563   assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
  564   return xmmreg_enc;
  565 }
  566 
  567 static int raw_encode(KRegister kreg) {
  568   assert(kreg == knoreg || kreg->is_valid(), "sanity");
  569   int kreg_enc = kreg->raw_encoding();
  570   assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
  571   return kreg_enc;
  572 }
  573 
  574 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
  575   return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
  576 }
  577 
  578 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
  579   return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
  580 }
  581 
  582 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
  583   assert((mod & 3) != 0b11, "forbidden");
  584   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  585   emit_int8(modrm);
  586 }
  587 
  588 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
  589                                         int disp) {
  590   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  591   emit_int16(modrm, disp & 0xFF);
  592 }
  593 
  594 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
  595                                       Address::ScaleFactor scale, int index_enc, int base_enc) {
  596   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  597   int sib = sib_encoding(scale, index_enc, base_enc);
  598   emit_int16(modrm, sib);
  599 }
  600 
  601 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
  602                                             Address::ScaleFactor scale, int index_enc, int base_enc,
  603                                             int disp) {
  604   int modrm = modrm_encoding(mod, dst_enc, src_enc);
  605   int sib = sib_encoding(scale, index_enc, base_enc);
  606   emit_int24(modrm, sib, disp & 0xFF);
  607 }
  608 
  609 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
  610                                     Address::ScaleFactor scale, int disp,
  611                                     RelocationHolder const& rspec,
  612                                     int post_addr_length) {
  613   bool no_relocation = (rspec.type() == relocInfo::none);
  614 
  615   if (is_valid_encoding(base_enc)) {
  616     if (is_valid_encoding(index_enc)) {
  617       assert(scale != Address::no_scale, "inconsistent address");
  618       // [base + index*scale + disp]
  619       if (disp == 0 && no_relocation &&
  620           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  621         // [base + index*scale]
  622         // [00 reg 100][ss index base]
  623         emit_modrm_sib(0b00, reg_enc, 0b100,
  624                        scale, index_enc, base_enc);
  625       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  626         // [base + index*scale + imm8]
  627         // [01 reg 100][ss index base] imm8
  628         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  629                              scale, index_enc, base_enc,
  630                              disp);
  631       } else {
  632         // [base + index*scale + disp32]
  633         // [10 reg 100][ss index base] disp32
  634         emit_modrm_sib(0b10, reg_enc, 0b100,
  635                        scale, index_enc, base_enc);
  636         emit_data(disp, rspec, disp32_operand);
  637       }
  638     } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
  639       // [rsp + disp]
  640       if (disp == 0 && no_relocation) {
  641         // [rsp]
  642         // [00 reg 100][00 100 100]
  643         emit_modrm_sib(0b00, reg_enc, 0b100,
  644                        Address::times_1, 0b100, 0b100);
  645       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  646         // [rsp + imm8]
  647         // [01 reg 100][00 100 100] disp8
  648         emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
  649                              Address::times_1, 0b100, 0b100,
  650                              disp);
  651       } else {
  652         // [rsp + imm32]
  653         // [10 reg 100][00 100 100] disp32
  654         emit_modrm_sib(0b10, reg_enc, 0b100,
  655                        Address::times_1, 0b100, 0b100);
  656         emit_data(disp, rspec, disp32_operand);
  657       }
  658     } else {
  659       // [base + disp]
  660       assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
  661       if (disp == 0 && no_relocation &&
  662           base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
  663         // [base]
  664         // [00 reg base]
  665         emit_modrm(0, reg_enc, base_enc);
  666       } else if (emit_compressed_disp_byte(disp) && no_relocation) {
  667         // [base + disp8]
  668         // [01 reg base] disp8
  669         emit_modrm_disp8(0b01, reg_enc, base_enc,
  670                          disp);
  671       } else {
  672         // [base + disp32]
  673         // [10 reg base] disp32
  674         emit_modrm(0b10, reg_enc, base_enc);
  675         emit_data(disp, rspec, disp32_operand);
  676       }
  677     }
  678   } else {
  679     if (is_valid_encoding(index_enc)) {
  680       assert(scale != Address::no_scale, "inconsistent address");
  681       // base == noreg
  682       // [index*scale + disp]
  683       // [00 reg 100][ss index 101] disp32
  684       emit_modrm_sib(0b00, reg_enc, 0b100,
  685                      scale, index_enc, 0b101 /* no base */);
  686       emit_data(disp, rspec, disp32_operand);
  687     } else if (!no_relocation) {
  688       // base == noreg, index == noreg
  689       // [disp] (64bit) RIP-RELATIVE (32bit) abs
  690       // [00 reg 101] disp32
  691 
  692       emit_modrm(0b00, reg_enc, 0b101 /* no base */);
  693       // Note that the RIP-rel. correction applies to the generated
  694       // disp field, but _not_ to the target address in the rspec.
  695 
  696       // disp was created by converting the target address minus the pc
  697       // at the start of the instruction. That needs more correction here.
  698       // intptr_t disp = target - next_ip;
  699       assert(inst_mark() != nullptr, "must be inside InstructionMark");
  700       address next_ip = pc() + sizeof(int32_t) + post_addr_length;
  701       int64_t adjusted = disp;
  702       // Do rip-rel adjustment for 64bit
  703       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
  704       assert(is_simm32(adjusted),
  705              "must be 32bit offset (RIP relative address)");
  706       emit_data((int32_t) adjusted, rspec, disp32_operand);
  707 
  708     } else {
  709       // base == noreg, index == noreg, no_relocation == true
  710       // 32bit never did this, did everything as the rip-rel/disp code above
  711       // [disp] ABSOLUTE
  712       // [00 reg 100][00 100 101] disp32
  713       emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
  714                      Address::times_1, 0b100, 0b101);
  715       emit_data(disp, rspec, disp32_operand);
  716     }
  717   }
  718 }
  719 
  720 void Assembler::emit_operand(Register reg, Register base, Register index,
  721                              Address::ScaleFactor scale, int disp,
  722                              RelocationHolder const& rspec,
  723                              int post_addr_length) {
  724   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  725   emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
  726                       scale, disp, rspec, post_addr_length);
  727 
  728 }
  729 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
  730                              Address::ScaleFactor scale, int disp,
  731                              RelocationHolder const& rspec,
  732                              int post_addr_length) {
  733   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  734   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  735   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
  736                       scale, disp, rspec, post_addr_length);
  737 }
  738 
  739 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
  740                              Address::ScaleFactor scale, int disp,
  741                              RelocationHolder const& rspec,
  742                              int post_addr_length) {
  743   assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
  744   assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
  745   emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
  746                       scale, disp, rspec, post_addr_length);
  747 }
  748 
  749 void Assembler::emit_operand(KRegister kreg, Address adr,
  750                              int post_addr_length) {
  751   emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
  752                adr._rspec,
  753                post_addr_length);
  754 }
  755 
  756 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
  757                              Address::ScaleFactor scale, int disp,
  758                              RelocationHolder const& rspec,
  759                              int post_addr_length) {
  760   assert(!index->is_valid() || index != rsp, "illegal addressing mode");
  761   emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
  762                       scale, disp, rspec, post_addr_length);
  763 }
  764 
  765 // Secret local extension to Assembler::WhichOperand:
  766 #define end_pc_operand (_WhichOperand_limit)
  767 
  768 address Assembler::locate_operand(address inst, WhichOperand which) {
  769   // Decode the given instruction, and return the address of
  770   // an embedded 32-bit operand word.
  771 
  772   // If "which" is disp32_operand, selects the displacement portion
  773   // of an effective address specifier.
  774   // If "which" is imm64_operand, selects the trailing immediate constant.
  775   // If "which" is call32_operand, selects the displacement of a call or jump.
  776   // Caller is responsible for ensuring that there is such an operand,
  777   // and that it is 32/64 bits wide.
  778 
  779   // If "which" is end_pc_operand, find the end of the instruction.
  780 
  781   address ip = inst;
  782   bool is_64bit = false;
  783 
  784   debug_only(bool has_disp32 = false);
  785   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
  786 
  787   again_after_prefix:
  788   switch (0xFF & *ip++) {
  789 
  790   // These convenience macros generate groups of "case" labels for the switch.
  791 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
  792 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
  793              case (x)+4: case (x)+5: case (x)+6: case (x)+7
  794 #define REP16(x) REP8((x)+0): \
  795               case REP8((x)+8)
  796 
  797   case CS_segment:
  798   case SS_segment:
  799   case DS_segment:
  800   case ES_segment:
  801   case FS_segment:
  802   case GS_segment:
  803     // Seems dubious
  804     LP64_ONLY(assert(false, "shouldn't have that prefix"));
  805     assert(ip == inst+1, "only one prefix allowed");
  806     goto again_after_prefix;
  807 
  808   case 0x67:
  809   case REX:
  810   case REX_B:
  811   case REX_X:
  812   case REX_XB:
  813   case REX_R:
  814   case REX_RB:
  815   case REX_RX:
  816   case REX_RXB:
  817     NOT_LP64(assert(false, "64bit prefixes"));
  818     goto again_after_prefix;
  819 
  820   case REX_W:
  821   case REX_WB:
  822   case REX_WX:
  823   case REX_WXB:
  824   case REX_WR:
  825   case REX_WRB:
  826   case REX_WRX:
  827   case REX_WRXB:
  828     NOT_LP64(assert(false, "64bit prefixes"));
  829     is_64bit = true;
  830     goto again_after_prefix;
  831 
  832   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
  833   case 0x88: // movb a, r
  834   case 0x89: // movl a, r
  835   case 0x8A: // movb r, a
  836   case 0x8B: // movl r, a
  837   case 0x8F: // popl a
  838     debug_only(has_disp32 = true);
  839     break;
  840 
  841   case 0x68: // pushq #32
  842     if (which == end_pc_operand) {
  843       return ip + 4;
  844     }
  845     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
  846     return ip;                  // not produced by emit_operand
  847 
  848   case 0x66: // movw ... (size prefix)
  849     again_after_size_prefix2:
  850     switch (0xFF & *ip++) {
  851     case REX:
  852     case REX_B:
  853     case REX_X:
  854     case REX_XB:
  855     case REX_R:
  856     case REX_RB:
  857     case REX_RX:
  858     case REX_RXB:
  859     case REX_W:
  860     case REX_WB:
  861     case REX_WX:
  862     case REX_WXB:
  863     case REX_WR:
  864     case REX_WRB:
  865     case REX_WRX:
  866     case REX_WRXB:
  867       NOT_LP64(assert(false, "64bit prefix found"));
  868       goto again_after_size_prefix2;
  869     case 0x8B: // movw r, a
  870     case 0x89: // movw a, r
  871       debug_only(has_disp32 = true);
  872       break;
  873     case 0xC7: // movw a, #16
  874       debug_only(has_disp32 = true);
  875       tail_size = 2;  // the imm16
  876       break;
  877     case 0x0F: // several SSE/SSE2 variants
  878       ip--;    // reparse the 0x0F
  879       goto again_after_prefix;
  880     default:
  881       ShouldNotReachHere();
  882     }
  883     break;
  884 
  885   case REP8(0xB8): // movl/q r, #32/#64(oop?)
  886     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
  887     // these asserts are somewhat nonsensical
  888 #ifndef _LP64
  889     assert(which == imm_operand || which == disp32_operand,
  890            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  891 #else
  892     assert((which == call32_operand || which == imm_operand) && is_64bit ||
  893            which == narrow_oop_operand && !is_64bit,
  894            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
  895 #endif // _LP64
  896     return ip;
  897 
  898   case 0x69: // imul r, a, #32
  899   case 0xC7: // movl a, #32(oop?)
  900     tail_size = 4;
  901     debug_only(has_disp32 = true); // has both kinds of operands!
  902     break;
  903 
  904   case 0x0F: // movx..., etc.
  905     switch (0xFF & *ip++) {
  906     case 0x3A: // pcmpestri
  907       tail_size = 1;
  908     case 0x38: // ptest, pmovzxbw
  909       ip++; // skip opcode
  910       debug_only(has_disp32 = true); // has both kinds of operands!
  911       break;
  912 
  913     case 0x70: // pshufd r, r/a, #8
  914       debug_only(has_disp32 = true); // has both kinds of operands!
  915     case 0x73: // psrldq r, #8
  916       tail_size = 1;
  917       break;
  918 
  919     case 0x10: // movups
  920     case 0x11: // movups
  921     case 0x12: // movlps
  922     case 0x28: // movaps
  923     case 0x2E: // ucomiss
  924     case 0x2F: // comiss
  925     case 0x54: // andps
  926     case 0x55: // andnps
  927     case 0x56: // orps
  928     case 0x57: // xorps
  929     case 0x58: // addpd
  930     case 0x59: // mulpd
  931     case 0x6E: // movd
  932     case 0x7E: // movd
  933     case 0x6F: // movdq
  934     case 0x7F: // movdq
  935     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
  936     case 0xFE: // paddd
  937       debug_only(has_disp32 = true);
  938       break;
  939 
  940     case 0xAD: // shrd r, a, %cl
  941     case 0xAF: // imul r, a
  942     case 0xBE: // movsbl r, a (movsxb)
  943     case 0xBF: // movswl r, a (movsxw)
  944     case 0xB6: // movzbl r, a (movzxb)
  945     case 0xB7: // movzwl r, a (movzxw)
  946     case REP16(0x40): // cmovl cc, r, a
  947     case 0xB0: // cmpxchgb
  948     case 0xB1: // cmpxchg
  949     case 0xC1: // xaddl
  950     case 0xC7: // cmpxchg8
  951     case REP16(0x90): // setcc a
  952       debug_only(has_disp32 = true);
  953       // fall out of the switch to decode the address
  954       break;
  955 
  956     case 0xC4: // pinsrw r, a, #8
  957       debug_only(has_disp32 = true);
  958     case 0xC5: // pextrw r, r, #8
  959       tail_size = 1;  // the imm8
  960       break;
  961 
  962     case 0xAC: // shrd r, a, #8
  963       debug_only(has_disp32 = true);
  964       tail_size = 1;  // the imm8
  965       break;
  966 
  967     case REP16(0x80): // jcc rdisp32
  968       if (which == end_pc_operand)  return ip + 4;
  969       assert(which == call32_operand, "jcc has no disp32 or imm");
  970       return ip;
  971     default:
  972       ShouldNotReachHere();
  973     }
  974     break;
  975 
  976   case 0x81: // addl a, #32; addl r, #32
  977     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  978     // on 32bit in the case of cmpl, the imm might be an oop
  979     tail_size = 4;
  980     debug_only(has_disp32 = true); // has both kinds of operands!
  981     break;
  982 
  983   case 0x83: // addl a, #8; addl r, #8
  984     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
  985     debug_only(has_disp32 = true); // has both kinds of operands!
  986     tail_size = 1;
  987     break;
  988 
  989   case 0x15: // adc rax, #32
  990   case 0x05: // add rax, #32
  991   case 0x25: // and rax, #32
  992   case 0x3D: // cmp rax, #32
  993   case 0x0D: // or  rax, #32
  994   case 0x1D: // sbb rax, #32
  995   case 0x2D: // sub rax, #32
  996   case 0x35: // xor rax, #32
  997     return which == end_pc_operand ? ip + 4 : ip;
  998 
  999   case 0x9B:
 1000     switch (0xFF & *ip++) {
 1001     case 0xD9: // fnstcw a
 1002       debug_only(has_disp32 = true);
 1003       break;
 1004     default:
 1005       ShouldNotReachHere();
 1006     }
 1007     break;
 1008 
 1009   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 1010   case REP4(0x10): // adc...
 1011   case REP4(0x20): // and...
 1012   case REP4(0x30): // xor...
 1013   case REP4(0x08): // or...
 1014   case REP4(0x18): // sbb...
 1015   case REP4(0x28): // sub...
 1016   case 0xF7: // mull a
 1017   case 0x8D: // lea r, a
 1018   case 0x87: // xchg r, a
 1019   case REP4(0x38): // cmp...
 1020   case 0x85: // test r, a
 1021     debug_only(has_disp32 = true); // has both kinds of operands!
 1022     break;
 1023 
 1024   case 0xA8: // testb rax, #8
 1025     return which == end_pc_operand ? ip + 1 : ip;
 1026   case 0xA9: // testl/testq rax, #32
 1027     return which == end_pc_operand ? ip + 4 : ip;
 1028 
 1029   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 1030   case 0xC6: // movb a, #8
 1031   case 0x80: // cmpb a, #8
 1032   case 0x6B: // imul r, a, #8
 1033     debug_only(has_disp32 = true); // has both kinds of operands!
 1034     tail_size = 1; // the imm8
 1035     break;
 1036 
 1037   case 0xC4: // VEX_3bytes
 1038   case 0xC5: // VEX_2bytes
 1039     assert((UseAVX > 0), "shouldn't have VEX prefix");
 1040     assert(ip == inst+1, "no prefixes allowed");
 1041     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 1042     // but they have prefix 0x0F and processed when 0x0F processed above.
 1043     //
 1044     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 1045     // instructions (these instructions are not supported in 64-bit mode).
 1046     // To distinguish them bits [7:6] are set in the VEX second byte since
 1047     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 1048     // those VEX bits REX and vvvv bits are inverted.
 1049     //
 1050     // Fortunately C2 doesn't generate these instructions so we don't need
 1051     // to check for them in product version.
 1052 
 1053     // Check second byte
 1054     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 1055 
 1056     int vex_opcode;
 1057     // First byte
 1058     if ((0xFF & *inst) == VEX_3bytes) {
 1059       vex_opcode = VEX_OPCODE_MASK & *ip;
 1060       ip++; // third byte
 1061       is_64bit = ((VEX_W & *ip) == VEX_W);
 1062     } else {
 1063       vex_opcode = VEX_OPCODE_0F;
 1064     }
 1065     ip++; // opcode
 1066     // To find the end of instruction (which == end_pc_operand).
 1067     switch (vex_opcode) {
 1068       case VEX_OPCODE_0F:
 1069         switch (0xFF & *ip) {
 1070         case 0x70: // pshufd r, r/a, #8
 1071         case 0x71: // ps[rl|ra|ll]w r, #8
 1072         case 0x72: // ps[rl|ra|ll]d r, #8
 1073         case 0x73: // ps[rl|ra|ll]q r, #8
 1074         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 1075         case 0xC4: // pinsrw r, r, r/a, #8
 1076         case 0xC5: // pextrw r/a, r, #8
 1077         case 0xC6: // shufp[s|d] r, r, r/a, #8
 1078           tail_size = 1;  // the imm8
 1079           break;
 1080         }
 1081         break;
 1082       case VEX_OPCODE_0F_3A:
 1083         tail_size = 1;
 1084         break;
 1085     }
 1086     ip++; // skip opcode
 1087     debug_only(has_disp32 = true); // has both kinds of operands!
 1088     break;
 1089 
 1090   case 0x62: // EVEX_4bytes
 1091     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 1092     assert(ip == inst+1, "no prefixes allowed");
 1093     // no EVEX collisions, all instructions that have 0x62 opcodes
 1094     // have EVEX versions and are subopcodes of 0x66
 1095     ip++; // skip P0 and examine W in P1
 1096     is_64bit = ((VEX_W & *ip) == VEX_W);
 1097     ip++; // move to P2
 1098     ip++; // skip P2, move to opcode
 1099     // To find the end of instruction (which == end_pc_operand).
 1100     switch (0xFF & *ip) {
 1101     case 0x22: // pinsrd r, r/a, #8
 1102     case 0x61: // pcmpestri r, r/a, #8
 1103     case 0x70: // pshufd r, r/a, #8
 1104     case 0x73: // psrldq r, #8
 1105     case 0x1f: // evpcmpd/evpcmpq
 1106     case 0x3f: // evpcmpb/evpcmpw
 1107       tail_size = 1;  // the imm8
 1108       break;
 1109     default:
 1110       break;
 1111     }
 1112     ip++; // skip opcode
 1113     debug_only(has_disp32 = true); // has both kinds of operands!
 1114     break;
 1115 
 1116   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 1117   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 1118   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 1119   case 0xDD: // fld_d a; fst_d a; fstp_d a
 1120   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 1121   case 0xDF: // fild_d a; fistp_d a
 1122   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 1123   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 1124   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 1125     debug_only(has_disp32 = true);
 1126     break;
 1127 
 1128   case 0xE8: // call rdisp32
 1129   case 0xE9: // jmp  rdisp32
 1130     if (which == end_pc_operand)  return ip + 4;
 1131     assert(which == call32_operand, "call has no disp32 or imm");
 1132     return ip;
 1133 
 1134   case 0xF0:                    // Lock
 1135     goto again_after_prefix;
 1136 
 1137   case 0xF3:                    // For SSE
 1138   case 0xF2:                    // For SSE2
 1139     switch (0xFF & *ip++) {
 1140     case REX:
 1141     case REX_B:
 1142     case REX_X:
 1143     case REX_XB:
 1144     case REX_R:
 1145     case REX_RB:
 1146     case REX_RX:
 1147     case REX_RXB:
 1148     case REX_W:
 1149     case REX_WB:
 1150     case REX_WX:
 1151     case REX_WXB:
 1152     case REX_WR:
 1153     case REX_WRB:
 1154     case REX_WRX:
 1155     case REX_WRXB:
 1156       NOT_LP64(assert(false, "found 64bit prefix"));
 1157       ip++;
 1158     default:
 1159       ip++;
 1160     }
 1161     debug_only(has_disp32 = true); // has both kinds of operands!
 1162     break;
 1163 
 1164   default:
 1165     ShouldNotReachHere();
 1166 
 1167 #undef REP8
 1168 #undef REP16
 1169   }
 1170 
 1171   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 1172 #ifdef _LP64
 1173   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 1174 #else
 1175   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 1176   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 1177 #endif // LP64
 1178   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 1179 
 1180   // parse the output of emit_operand
 1181   int op2 = 0xFF & *ip++;
 1182   int base = op2 & 0x07;
 1183   int op3 = -1;
 1184   const int b100 = 4;
 1185   const int b101 = 5;
 1186   if (base == b100 && (op2 >> 6) != 3) {
 1187     op3 = 0xFF & *ip++;
 1188     base = op3 & 0x07;   // refetch the base
 1189   }
 1190   // now ip points at the disp (if any)
 1191 
 1192   switch (op2 >> 6) {
 1193   case 0:
 1194     // [00 reg  100][ss index base]
 1195     // [00 reg  100][00   100  esp]
 1196     // [00 reg base]
 1197     // [00 reg  100][ss index  101][disp32]
 1198     // [00 reg  101]               [disp32]
 1199 
 1200     if (base == b101) {
 1201       if (which == disp32_operand)
 1202         return ip;              // caller wants the disp32
 1203       ip += 4;                  // skip the disp32
 1204     }
 1205     break;
 1206 
 1207   case 1:
 1208     // [01 reg  100][ss index base][disp8]
 1209     // [01 reg  100][00   100  esp][disp8]
 1210     // [01 reg base]               [disp8]
 1211     ip += 1;                    // skip the disp8
 1212     break;
 1213 
 1214   case 2:
 1215     // [10 reg  100][ss index base][disp32]
 1216     // [10 reg  100][00   100  esp][disp32]
 1217     // [10 reg base]               [disp32]
 1218     if (which == disp32_operand)
 1219       return ip;                // caller wants the disp32
 1220     ip += 4;                    // skip the disp32
 1221     break;
 1222 
 1223   case 3:
 1224     // [11 reg base]  (not a memory addressing mode)
 1225     break;
 1226   }
 1227 
 1228   if (which == end_pc_operand) {
 1229     return ip + tail_size;
 1230   }
 1231 
 1232 #ifdef _LP64
 1233   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 1234 #else
 1235   assert(which == imm_operand, "instruction has only an imm field");
 1236 #endif // LP64
 1237   return ip;
 1238 }
 1239 
 1240 address Assembler::locate_next_instruction(address inst) {
 1241   // Secretly share code with locate_operand:
 1242   return locate_operand(inst, end_pc_operand);
 1243 }
 1244 
 1245 
 1246 #ifdef ASSERT
 1247 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 1248   address inst = inst_mark();
 1249   assert(inst != nullptr && inst < pc(), "must point to beginning of instruction");
 1250   address opnd;
 1251 
 1252   Relocation* r = rspec.reloc();
 1253   if (r->type() == relocInfo::none) {
 1254     return;
 1255   } else if (r->is_call() || format == call32_operand) {
 1256     // assert(format == imm32_operand, "cannot specify a nonzero format");
 1257     opnd = locate_operand(inst, call32_operand);
 1258   } else if (r->is_data()) {
 1259     assert(format == imm_operand || format == disp32_operand
 1260            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 1261     opnd = locate_operand(inst, (WhichOperand)format);
 1262   } else {
 1263     assert(format == imm_operand, "cannot specify a format");
 1264     return;
 1265   }
 1266   assert(opnd == pc(), "must put operand where relocs can find it");
 1267 }
 1268 #endif // ASSERT
 1269 
 1270 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
 1271   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1272 }
 1273 
 1274 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
 1275   if (adr.isxmmindex()) {
 1276      emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1277   } else {
 1278      emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
 1279   }
 1280 }
 1281 
 1282 // Now the Assembler instructions (identical for 32/64 bits)
 1283 
 1284 void Assembler::adcl(Address dst, int32_t imm32) {
 1285   InstructionMark im(this);
 1286   prefix(dst);
 1287   emit_arith_operand(0x81, rdx, dst, imm32);
 1288 }
 1289 
 1290 void Assembler::adcl(Address dst, Register src) {
 1291   InstructionMark im(this);
 1292   prefix(dst, src);
 1293   emit_int8(0x11);
 1294   emit_operand(src, dst, 0);
 1295 }
 1296 
 1297 void Assembler::adcl(Register dst, int32_t imm32) {
 1298   prefix(dst);
 1299   emit_arith(0x81, 0xD0, dst, imm32);
 1300 }
 1301 
 1302 void Assembler::adcl(Register dst, Address src) {
 1303   InstructionMark im(this);
 1304   prefix(src, dst);
 1305   emit_int8(0x13);
 1306   emit_operand(dst, src, 0);
 1307 }
 1308 
 1309 void Assembler::adcl(Register dst, Register src) {
 1310   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1311   emit_arith(0x13, 0xC0, dst, src);
 1312 }
 1313 
 1314 void Assembler::addl(Address dst, int32_t imm32) {
 1315   InstructionMark im(this);
 1316   prefix(dst);
 1317   emit_arith_operand(0x81, rax, dst, imm32);
 1318 }
 1319 
 1320 void Assembler::addb(Address dst, int imm8) {
 1321   InstructionMark im(this);
 1322   prefix(dst);
 1323   emit_int8((unsigned char)0x80);
 1324   emit_operand(rax, dst, 1);
 1325   emit_int8(imm8);
 1326 }
 1327 
 1328 void Assembler::addb(Address dst, Register src) {
 1329   InstructionMark im(this);
 1330   prefix(dst, src);
 1331   emit_int8(0x00);
 1332   emit_operand(src, dst, 0);
 1333 }
 1334 
 1335 void Assembler::addw(Register dst, Register src) {
 1336   emit_int8(0x66);
 1337   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1338   emit_arith(0x03, 0xC0, dst, src);
 1339 }
 1340 
 1341 void Assembler::addw(Address dst, int imm16) {
 1342   InstructionMark im(this);
 1343   emit_int8(0x66);
 1344   prefix(dst);
 1345   emit_int8((unsigned char)0x81);
 1346   emit_operand(rax, dst, 2);
 1347   emit_int16(imm16);
 1348 }
 1349 
 1350 void Assembler::addw(Address dst, Register src) {
 1351   InstructionMark im(this);
 1352   emit_int8(0x66);
 1353   prefix(dst, src);
 1354   emit_int8(0x01);
 1355   emit_operand(src, dst, 0);
 1356 }
 1357 
 1358 void Assembler::addl(Address dst, Register src) {
 1359   InstructionMark im(this);
 1360   prefix(dst, src);
 1361   emit_int8(0x01);
 1362   emit_operand(src, dst, 0);
 1363 }
 1364 
 1365 void Assembler::addl(Register dst, int32_t imm32) {
 1366   prefix(dst);
 1367   emit_arith(0x81, 0xC0, dst, imm32);
 1368 }
 1369 
 1370 void Assembler::addl(Register dst, Address src) {
 1371   InstructionMark im(this);
 1372   prefix(src, dst);
 1373   emit_int8(0x03);
 1374   emit_operand(dst, src, 0);
 1375 }
 1376 
 1377 void Assembler::addl(Register dst, Register src) {
 1378   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1379   emit_arith(0x03, 0xC0, dst, src);
 1380 }
 1381 
 1382 void Assembler::addr_nop_4() {
 1383   assert(UseAddressNop, "no CPU support");
 1384   // 4 bytes: NOP DWORD PTR [EAX+0]
 1385   emit_int32(0x0F,
 1386              0x1F,
 1387              0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 1388              0);   // 8-bits offset (1 byte)
 1389 }
 1390 
 1391 void Assembler::addr_nop_5() {
 1392   assert(UseAddressNop, "no CPU support");
 1393   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 1394   emit_int32(0x0F,
 1395              0x1F,
 1396              0x44,  // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 1397              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1398   emit_int8(0);     // 8-bits offset (1 byte)
 1399 }
 1400 
 1401 void Assembler::addr_nop_7() {
 1402   assert(UseAddressNop, "no CPU support");
 1403   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 1404   emit_int24(0x0F,
 1405              0x1F,
 1406              (unsigned char)0x80);
 1407                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 1408   emit_int32(0);   // 32-bits offset (4 bytes)
 1409 }
 1410 
 1411 void Assembler::addr_nop_8() {
 1412   assert(UseAddressNop, "no CPU support");
 1413   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 1414   emit_int32(0x0F,
 1415              0x1F,
 1416              (unsigned char)0x84,
 1417                     // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 1418              0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 1419   emit_int32(0);    // 32-bits offset (4 bytes)
 1420 }
 1421 
 1422 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 1423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1424   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1425   attributes.set_rex_vex_w_reverted();
 1426   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1427   emit_int16(0x58, (0xC0 | encode));
 1428 }
 1429 
 1430 void Assembler::addsd(XMMRegister dst, Address src) {
 1431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1432   InstructionMark im(this);
 1433   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1434   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1435   attributes.set_rex_vex_w_reverted();
 1436   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 1437   emit_int8(0x58);
 1438   emit_operand(dst, src, 0);
 1439 }
 1440 
 1441 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 1442   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1443   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1444   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1445   emit_int16(0x58, (0xC0 | encode));
 1446 }
 1447 
 1448 void Assembler::addss(XMMRegister dst, Address src) {
 1449   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1450   InstructionMark im(this);
 1451   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1452   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1453   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1454   emit_int8(0x58);
 1455   emit_operand(dst, src, 0);
 1456 }
 1457 
 1458 void Assembler::aesdec(XMMRegister dst, Address src) {
 1459   assert(VM_Version::supports_aes(), "");
 1460   InstructionMark im(this);
 1461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1462   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1463   emit_int8((unsigned char)0xDE);
 1464   emit_operand(dst, src, 0);
 1465 }
 1466 
 1467 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
 1468   assert(VM_Version::supports_aes(), "");
 1469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1470   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1471   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1472 }
 1473 
 1474 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1475   assert(VM_Version::supports_avx512_vaes(), "");
 1476   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1477   attributes.set_is_evex_instruction();
 1478   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1479   emit_int16((unsigned char)0xDE, (0xC0 | encode));
 1480 }
 1481 
 1482 
 1483 void Assembler::aesdeclast(XMMRegister dst, Address src) {
 1484   assert(VM_Version::supports_aes(), "");
 1485   InstructionMark im(this);
 1486   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1487   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1488   emit_int8((unsigned char)0xDF);
 1489   emit_operand(dst, src, 0);
 1490 }
 1491 
 1492 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
 1493   assert(VM_Version::supports_aes(), "");
 1494   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1495   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1496   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1497 }
 1498 
 1499 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1500   assert(VM_Version::supports_avx512_vaes(), "");
 1501   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1502   attributes.set_is_evex_instruction();
 1503   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1504   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 1505 }
 1506 
 1507 void Assembler::aesenc(XMMRegister dst, Address src) {
 1508   assert(VM_Version::supports_aes(), "");
 1509   InstructionMark im(this);
 1510   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1511   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1512   emit_int8((unsigned char)0xDC);
 1513   emit_operand(dst, src, 0);
 1514 }
 1515 
 1516 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
 1517   assert(VM_Version::supports_aes(), "");
 1518   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1519   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1520   emit_int16((unsigned char)0xDC, 0xC0 | encode);
 1521 }
 1522 
 1523 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1524   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1526   attributes.set_is_evex_instruction();
 1527   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1528   emit_int16((unsigned char)0xDC, (0xC0 | encode));
 1529 }
 1530 
 1531 void Assembler::aesenclast(XMMRegister dst, Address src) {
 1532   assert(VM_Version::supports_aes(), "");
 1533   InstructionMark im(this);
 1534   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1535   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1536   emit_int8((unsigned char)0xDD);
 1537   emit_operand(dst, src, 0);
 1538 }
 1539 
 1540 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
 1541   assert(VM_Version::supports_aes(), "");
 1542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1543   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1544   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1545 }
 1546 
 1547 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 1548   assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
 1549   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1550   attributes.set_is_evex_instruction();
 1551   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1552   emit_int16((unsigned char)0xDD, (0xC0 | encode));
 1553 }
 1554 
 1555 void Assembler::andb(Address dst, Register src) {
 1556   InstructionMark im(this);
 1557   prefix(dst, src, true);
 1558   emit_int8(0x20);
 1559   emit_operand(src, dst, 0);
 1560 }
 1561 
 1562 void Assembler::andw(Register dst, Register src) {
 1563   (void)prefix_and_encode(dst->encoding(), src->encoding());
 1564   emit_arith(0x23, 0xC0, dst, src);
 1565 }
 1566 
 1567 void Assembler::andl(Address dst, int32_t imm32) {
 1568   InstructionMark im(this);
 1569   prefix(dst);
 1570   emit_arith_operand(0x81, as_Register(4), dst, imm32);
 1571 }
 1572 
 1573 void Assembler::andl(Register dst, int32_t imm32) {
 1574   prefix(dst);
 1575   emit_arith(0x81, 0xE0, dst, imm32);
 1576 }
 1577 
 1578 void Assembler::andl(Address dst, Register src) {
 1579   InstructionMark im(this);
 1580   prefix(dst, src);
 1581   emit_int8(0x21);
 1582   emit_operand(src, dst, 0);
 1583 }
 1584 
 1585 void Assembler::andl(Register dst, Address src) {
 1586   InstructionMark im(this);
 1587   prefix(src, dst);
 1588   emit_int8(0x23);
 1589   emit_operand(dst, src, 0);
 1590 }
 1591 
 1592 void Assembler::andl(Register dst, Register src) {
 1593   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1594   emit_arith(0x23, 0xC0, dst, src);
 1595 }
 1596 
 1597 void Assembler::andnl(Register dst, Register src1, Register src2) {
 1598   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1599   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1600   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1601   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 1602 }
 1603 
 1604 void Assembler::andnl(Register dst, Register src1, Address src2) {
 1605   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1606   InstructionMark im(this);
 1607   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1608   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1609   emit_int8((unsigned char)0xF2);
 1610   emit_operand(dst, src2, 0);
 1611 }
 1612 
 1613 void Assembler::bsfl(Register dst, Register src) {
 1614   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1615   emit_int24(0x0F,
 1616              (unsigned char)0xBC,
 1617              0xC0 | encode);
 1618 }
 1619 
 1620 void Assembler::bsrl(Register dst, Register src) {
 1621   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1622   emit_int24(0x0F,
 1623              (unsigned char)0xBD,
 1624              0xC0 | encode);
 1625 }
 1626 
 1627 void Assembler::bswapl(Register reg) { // bswap
 1628   int encode = prefix_and_encode(reg->encoding());
 1629   emit_int16(0x0F, (0xC8 | encode));
 1630 }
 1631 
 1632 void Assembler::blsil(Register dst, Register src) {
 1633   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1634   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1635   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1636   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1637 }
 1638 
 1639 void Assembler::blsil(Register dst, Address src) {
 1640   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1641   InstructionMark im(this);
 1642   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1643   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1644   emit_int8((unsigned char)0xF3);
 1645   emit_operand(rbx, src, 0);
 1646 }
 1647 
 1648 void Assembler::blsmskl(Register dst, Register src) {
 1649   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1650   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1651   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1652   emit_int16((unsigned char)0xF3,
 1653              0xC0 | encode);
 1654 }
 1655 
 1656 void Assembler::blsmskl(Register dst, Address src) {
 1657   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1658   InstructionMark im(this);
 1659   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1660   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1661   emit_int8((unsigned char)0xF3);
 1662   emit_operand(rdx, src, 0);
 1663 }
 1664 
 1665 void Assembler::blsrl(Register dst, Register src) {
 1666   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1667   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1668   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1669   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 1670 }
 1671 
 1672 void Assembler::blsrl(Register dst, Address src) {
 1673   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
 1674   InstructionMark im(this);
 1675   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 1676   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
 1677   emit_int8((unsigned char)0xF3);
 1678   emit_operand(rcx, src, 0);
 1679 }
 1680 
 1681 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 1682   // suspect disp32 is always good
 1683   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 1684 
 1685   if (L.is_bound()) {
 1686     const int long_size = 5;
 1687     int offs = (int)( target(L) - pc() );
 1688     assert(offs <= 0, "assembler error");
 1689     InstructionMark im(this);
 1690     // 1110 1000 #32-bit disp
 1691     emit_int8((unsigned char)0xE8);
 1692     emit_data(offs - long_size, rtype, operand);
 1693   } else {
 1694     InstructionMark im(this);
 1695     // 1110 1000 #32-bit disp
 1696     L.add_patch_at(code(), locator());
 1697 
 1698     emit_int8((unsigned char)0xE8);
 1699     emit_data(int(0), rtype, operand);
 1700   }
 1701 }
 1702 
 1703 void Assembler::call(Register dst) {
 1704   int encode = prefix_and_encode(dst->encoding());
 1705   emit_int16((unsigned char)0xFF, (0xD0 | encode));
 1706 }
 1707 
 1708 
 1709 void Assembler::call(Address adr) {
 1710   InstructionMark im(this);
 1711   prefix(adr);
 1712   emit_int8((unsigned char)0xFF);
 1713   emit_operand(rdx, adr, 0);
 1714 }
 1715 
 1716 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
 1717   InstructionMark im(this);
 1718   emit_int8((unsigned char)0xE8);
 1719   intptr_t disp = entry - (pc() + sizeof(int32_t));
 1720   // Entry is null in case of a scratch emit.
 1721   assert(entry == nullptr || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
 1722   // Technically, should use call32_operand, but this format is
 1723   // implied by the fact that we're emitting a call instruction.
 1724 
 1725   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
 1726   emit_data((int) disp, rspec, operand);
 1727 }
 1728 
 1729 void Assembler::cdql() {
 1730   emit_int8((unsigned char)0x99);
 1731 }
 1732 
 1733 void Assembler::cld() {
 1734   emit_int8((unsigned char)0xFC);
 1735 }
 1736 
 1737 void Assembler::cmovl(Condition cc, Register dst, Register src) {
 1738   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1739   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 1740   emit_int24(0x0F,
 1741              0x40 | cc,
 1742              0xC0 | encode);
 1743 }
 1744 
 1745 
 1746 void Assembler::cmovl(Condition cc, Register dst, Address src) {
 1747   InstructionMark im(this);
 1748   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
 1749   prefix(src, dst);
 1750   emit_int16(0x0F, (0x40 | cc));
 1751   emit_operand(dst, src, 0);
 1752 }
 1753 
 1754 void Assembler::cmpb(Address dst, int imm8) {
 1755   InstructionMark im(this);
 1756   prefix(dst);
 1757   emit_int8((unsigned char)0x80);
 1758   emit_operand(rdi, dst, 1);
 1759   emit_int8(imm8);
 1760 }
 1761 
 1762 void Assembler::cmpl(Address dst, int32_t imm32) {
 1763   InstructionMark im(this);
 1764   prefix(dst);
 1765   emit_arith_operand(0x81, as_Register(7), dst, imm32);
 1766 }
 1767 
 1768 void Assembler::cmpl(Register dst, int32_t imm32) {
 1769   prefix(dst);
 1770   emit_arith(0x81, 0xF8, dst, imm32);
 1771 }
 1772 
 1773 void Assembler::cmpl(Register dst, Register src) {
 1774   (void) prefix_and_encode(dst->encoding(), src->encoding());
 1775   emit_arith(0x3B, 0xC0, dst, src);
 1776 }
 1777 
 1778 void Assembler::cmpl(Register dst, Address src) {
 1779   InstructionMark im(this);
 1780   prefix(src, dst);
 1781   emit_int8(0x3B);
 1782   emit_operand(dst, src, 0);
 1783 }
 1784 
 1785 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
 1786   InstructionMark im(this);
 1787   prefix(dst);
 1788   emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
 1789 }
 1790 
 1791 void Assembler::cmpw(Address dst, int imm16) {
 1792   InstructionMark im(this);
 1793   emit_int8(0x66);
 1794   prefix(dst);
 1795   emit_int8((unsigned char)0x81);
 1796   emit_operand(rdi, dst, 2);
 1797   emit_int16(imm16);
 1798 }
 1799 
 1800 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 1801 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1802 // The ZF is set if the compared values were equal, and cleared otherwise.
 1803 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
 1804   InstructionMark im(this);
 1805   prefix(adr, reg);
 1806   emit_int16(0x0F, (unsigned char)0xB1);
 1807   emit_operand(reg, adr, 0);
 1808 }
 1809 
 1810 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
 1811   InstructionMark im(this);
 1812   size_prefix();
 1813   prefix(adr, reg);
 1814   emit_int16(0x0F, (unsigned char)0xB1);
 1815   emit_operand(reg, adr, 0);
 1816 }
 1817 
 1818 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
 1819 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 1820 // The ZF is set if the compared values were equal, and cleared otherwise.
 1821 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
 1822   InstructionMark im(this);
 1823   prefix(adr, reg, true);
 1824   emit_int16(0x0F, (unsigned char)0xB0);
 1825   emit_operand(reg, adr, 0);
 1826 }
 1827 
 1828 void Assembler::comisd(XMMRegister dst, Address src) {
 1829   // NOTE: dbx seems to decode this as comiss even though the
 1830   // 0x66 is there. Strangely ucomisd comes out correct
 1831   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1832   InstructionMark im(this);
 1833   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
 1834   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 1835   attributes.set_rex_vex_w_reverted();
 1836   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1837   emit_int8(0x2F);
 1838   emit_operand(dst, src, 0);
 1839 }
 1840 
 1841 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 1842   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1843   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1844   attributes.set_rex_vex_w_reverted();
 1845   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 1846   emit_int16(0x2F, (0xC0 | encode));
 1847 }
 1848 
 1849 void Assembler::comiss(XMMRegister dst, Address src) {
 1850   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1851   InstructionMark im(this);
 1852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1853   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 1854   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1855   emit_int8(0x2F);
 1856   emit_operand(dst, src, 0);
 1857 }
 1858 
 1859 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 1860   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 1861   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 1862   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 1863   emit_int16(0x2F, (0xC0 | encode));
 1864 }
 1865 
 1866 void Assembler::cpuid() {
 1867   emit_int16(0x0F, (unsigned char)0xA2);
 1868 }
 1869 
 1870 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
 1871 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
 1872 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1873 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
 1874 //
 1875 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
 1876 //
 1877 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
 1878 //
 1879 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
 1880 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
 1881   assert(VM_Version::supports_sse4_2(), "");
 1882   int8_t w = 0x01;
 1883   Prefix p = Prefix_EMPTY;
 1884 
 1885   emit_int8((unsigned char)0xF2);
 1886   switch (sizeInBytes) {
 1887   case 1:
 1888     w = 0;
 1889     break;
 1890   case 2:
 1891   case 4:
 1892     break;
 1893   LP64_ONLY(case 8:)
 1894     // This instruction is not valid in 32 bits
 1895     // Note:
 1896     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
 1897     //
 1898     // Page B - 72   Vol. 2C says
 1899     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
 1900     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
 1901     //                                                                            F0!!!
 1902     // while 3 - 208 Vol. 2A
 1903     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
 1904     //
 1905     // the 0 on a last bit is reserved for a different flavor of this instruction :
 1906     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
 1907     p = REX_W;
 1908     break;
 1909   default:
 1910     assert(0, "Unsupported value for a sizeInBytes argument");
 1911     break;
 1912   }
 1913   LP64_ONLY(prefix(crc, v, p);)
 1914   emit_int32(0x0F,
 1915              0x38,
 1916              0xF0 | w,
 1917              0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
 1918 }
 1919 
 1920 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
 1921   assert(VM_Version::supports_sse4_2(), "");
 1922   InstructionMark im(this);
 1923   int8_t w = 0x01;
 1924   Prefix p = Prefix_EMPTY;
 1925 
 1926   emit_int8((uint8_t)0xF2);
 1927   switch (sizeInBytes) {
 1928   case 1:
 1929     w = 0;
 1930     break;
 1931   case 2:
 1932   case 4:
 1933     break;
 1934   LP64_ONLY(case 8:)
 1935     // This instruction is not valid in 32 bits
 1936     p = REX_W;
 1937     break;
 1938   default:
 1939     assert(0, "Unsupported value for a sizeInBytes argument");
 1940     break;
 1941   }
 1942   LP64_ONLY(prefix(crc, adr, p);)
 1943   emit_int24(0x0F, 0x38, (0xF0 | w));
 1944   emit_operand(crc, adr, 0);
 1945 }
 1946 
 1947 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 1948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 1949   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1950   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1951   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1952 }
 1953 
 1954 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 1955   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 1956   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 1957   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 1958   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 1959 }
 1960 
 1961 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 1962   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1963   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1964   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1965   emit_int24(0x1D, (0xC0 | encode), imm8);
 1966 }
 1967 
 1968 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
 1969   assert(VM_Version::supports_evex(), "");
 1970   InstructionMark im(this);
 1971   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 1972   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
 1973   attributes.reset_is_clear_context();
 1974   attributes.set_embedded_opmask_register_specifier(mask);
 1975   attributes.set_is_evex_instruction();
 1976   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1977   emit_int8(0x1D);
 1978   emit_operand(src, dst, 1);
 1979   emit_int8(imm8);
 1980 }
 1981 
 1982 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) {
 1983   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1984   InstructionMark im(this);
 1985   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
 1986   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 1987   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 1988   emit_int8(0x1D);
 1989   emit_operand(src, dst, 1);
 1990   emit_int8(imm8);
 1991 }
 1992 
 1993 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 1994   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 1995   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
 1996   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 1997   emit_int16(0x13, (0xC0 | encode));
 1998 }
 1999 
 2000 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) {
 2001   assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
 2002   InstructionMark im(this);
 2003   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
 2004   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 2005   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2006   emit_int8(0x13);
 2007   emit_operand(dst, src, 0);
 2008 }
 2009 
 2010 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 2011   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2013   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2014   emit_int16(0x5B, (0xC0 | encode));
 2015 }
 2016 
 2017 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2018   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2019   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2020   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2021   emit_int16(0x5B, (0xC0 | encode));
 2022 }
 2023 
 2024 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 2025   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2026   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2027   attributes.set_rex_vex_w_reverted();
 2028   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2029   emit_int16(0x5A, (0xC0 | encode));
 2030 }
 2031 
 2032 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 2033   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2034   InstructionMark im(this);
 2035   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2036   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2037   attributes.set_rex_vex_w_reverted();
 2038   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2039   emit_int8(0x5A);
 2040   emit_operand(dst, src, 0);
 2041 }
 2042 
 2043 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 2044   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2046   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2047   emit_int16(0x2A, (0xC0 | encode));
 2048 }
 2049 
 2050 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 2051   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2052   InstructionMark im(this);
 2053   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2054   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2055   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2056   emit_int8(0x2A);
 2057   emit_operand(dst, src, 0);
 2058 }
 2059 
 2060 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 2061   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2063   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2064   emit_int16(0x2A, (0xC0 | encode));
 2065 }
 2066 
 2067 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 2068   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2069   InstructionMark im(this);
 2070   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2071   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2072   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2073   emit_int8(0x2A);
 2074   emit_operand(dst, src, 0);
 2075 }
 2076 
 2077 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 2078   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2079   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2080   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2081   emit_int16(0x2A, (0xC0 | encode));
 2082 }
 2083 
 2084 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 2085   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2086   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2087   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2088   emit_int16(0x5A, (0xC0 | encode));
 2089 }
 2090 
 2091 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 2092   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2093   InstructionMark im(this);
 2094   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2095   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2096   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2097   emit_int8(0x5A);
 2098   emit_operand(dst, src, 0);
 2099 }
 2100 
 2101 
 2102 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 2103   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2104   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2105   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2106   emit_int16(0x2C, (0xC0 | encode));
 2107 }
 2108 
 2109 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
 2110   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2112   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2113   emit_int16(0x2D, (0xC0 | encode));
 2114 }
 2115 
 2116 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 2117   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2118   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2119   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2120   emit_int16(0x2C, (0xC0 | encode));
 2121 }
 2122 
 2123 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
 2124   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2125   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2126   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2127   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2128   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2129 }
 2130 
 2131 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
 2132   assert(VM_Version::supports_ssse3(), "");
 2133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2134   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2135   emit_int16(0x1C, (0xC0 | encode));
 2136 }
 2137 
 2138 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
 2139   assert(VM_Version::supports_ssse3(), "");
 2140   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2141   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2142   emit_int16(0x1D, (0xC0 | encode));
 2143 }
 2144 
 2145 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
 2146   assert(VM_Version::supports_ssse3(), "");
 2147   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2148   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2149   emit_int16(0x1E, (0xC0 | encode));
 2150 }
 2151 
 2152 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
 2153   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2154          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2155          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
 2156   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2157   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2158   emit_int16(0x1C, (0xC0 | encode));
 2159 }
 2160 
 2161 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
 2162   assert(vector_len == AVX_128bit ? VM_Version::supports_avx()      :
 2163          vector_len == AVX_256bit ? VM_Version::supports_avx2()     :
 2164          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
 2165   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 2166   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2167   emit_int16(0x1D, (0xC0 | encode));
 2168 }
 2169 
 2170 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
 2171   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 2172   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 2173   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
 2174   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2175   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2176   emit_int16(0x1E, (0xC0 | encode));
 2177 }
 2178 
 2179 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
 2180   assert(UseAVX > 2, "");
 2181   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2182   attributes.set_is_evex_instruction();
 2183   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 2184   emit_int16(0x1F, (0xC0 | encode));
 2185 }
 2186 
 2187 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2188   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2189   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2190   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2191   emit_int16(0x5A, (0xC0 | encode));
 2192 }
 2193 
 2194 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2195   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2196   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2197   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2198   attributes.set_rex_vex_w_reverted();
 2199   emit_int16(0x5A, (0xC0 | encode));
 2200 }
 2201 
 2202 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2203   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2204   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2205   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2206   emit_int16(0x5B, (0xC0 | encode));
 2207 }
 2208 
 2209 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2210   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2211   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2212   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2213   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2214 }
 2215 
 2216 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
 2217   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 2218   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2219   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2220   emit_int16(0x5B, (0xC0 | encode));
 2221 }
 2222 
 2223 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2224   assert(VM_Version::supports_avx512dq(), "");
 2225   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2226   attributes.set_is_evex_instruction();
 2227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2228   emit_int16(0x7A, (0xC0 | encode));
 2229 }
 2230 
 2231 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2232   assert(VM_Version::supports_avx512dq(), "");
 2233   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2234   attributes.set_is_evex_instruction();
 2235   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2236   emit_int16(0x7B, (0xC0 | encode));
 2237 }
 2238 
 2239 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
 2240   assert(VM_Version::supports_avx512dq(), "");
 2241   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2242   attributes.set_is_evex_instruction();
 2243   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2244   emit_int16(0x5B, (0xC0 | encode));
 2245 }
 2246 
 2247 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
 2248   assert(VM_Version::supports_avx512dq(), "");
 2249   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2250   attributes.set_is_evex_instruction();
 2251   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2252   emit_int16(0x7A, (0xC0 | encode));
 2253 }
 2254 
 2255 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
 2256   assert(VM_Version::supports_avx512dq(), "");
 2257   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2258   attributes.set_is_evex_instruction();
 2259   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2260   emit_int16((unsigned char)0xE6, (0xC0 | encode));
 2261 }
 2262 
 2263 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
 2264   assert(VM_Version::supports_avx512bw(), "");
 2265   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2266   attributes.set_is_evex_instruction();
 2267   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2268   emit_int16(0x30, (0xC0 | encode));
 2269 }
 2270 
 2271 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
 2272   assert(UseAVX > 2, "");
 2273   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2274   attributes.set_is_evex_instruction();
 2275   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2276   emit_int16(0x33, (0xC0 | encode));
 2277 }
 2278 
 2279 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
 2280   assert(UseAVX > 2, "");
 2281   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2282   attributes.set_is_evex_instruction();
 2283   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2284   emit_int16(0x31, (0xC0 | encode));
 2285 }
 2286 
 2287 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2288   assert(UseAVX > 2, "");
 2289   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2290   attributes.set_is_evex_instruction();
 2291   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2292   emit_int16(0x35, (0xC0 | encode));
 2293 }
 2294 
 2295 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
 2296   assert(UseAVX > 2, "");
 2297   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2298   attributes.set_is_evex_instruction();
 2299   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2300   emit_int16(0x32, (0xC0 | encode));
 2301 }
 2302 
 2303 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
 2304   assert(UseAVX > 2, "");
 2305   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2306   attributes.set_is_evex_instruction();
 2307   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2308   emit_int16(0x34, (0xC0 | encode));
 2309 }
 2310 
 2311 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
 2312   assert(UseAVX > 2, "");
 2313   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2314   attributes.set_is_evex_instruction();
 2315   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 2316   emit_int16(0x25, (0xC0 | encode));
 2317 }
 2318 
 2319 void Assembler::decl(Address dst) {
 2320   // Don't use it directly. Use MacroAssembler::decrement() instead.
 2321   InstructionMark im(this);
 2322   prefix(dst);
 2323   emit_int8((unsigned char)0xFF);
 2324   emit_operand(rcx, dst, 0);
 2325 }
 2326 
 2327 void Assembler::divsd(XMMRegister dst, Address src) {
 2328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2329   InstructionMark im(this);
 2330   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2331   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 2332   attributes.set_rex_vex_w_reverted();
 2333   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2334   emit_int8(0x5E);
 2335   emit_operand(dst, src, 0);
 2336 }
 2337 
 2338 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 2339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2340   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2341   attributes.set_rex_vex_w_reverted();
 2342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2343   emit_int16(0x5E, (0xC0 | encode));
 2344 }
 2345 
 2346 void Assembler::divss(XMMRegister dst, Address src) {
 2347   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2348   InstructionMark im(this);
 2349   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2350   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 2351   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2352   emit_int8(0x5E);
 2353   emit_operand(dst, src, 0);
 2354 }
 2355 
 2356 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 2357   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2358   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 2359   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 2360   emit_int16(0x5E, (0xC0 | encode));
 2361 }
 2362 
 2363 void Assembler::hlt() {
 2364   emit_int8((unsigned char)0xF4);
 2365 }
 2366 
 2367 void Assembler::idivl(Register src) {
 2368   int encode = prefix_and_encode(src->encoding());
 2369   emit_int16((unsigned char)0xF7, (0xF8 | encode));
 2370 }
 2371 
 2372 void Assembler::divl(Register src) { // Unsigned
 2373   int encode = prefix_and_encode(src->encoding());
 2374   emit_int16((unsigned char)0xF7, (0xF0 | encode));
 2375 }
 2376 
 2377 void Assembler::imull(Register src) {
 2378   int encode = prefix_and_encode(src->encoding());
 2379   emit_int16((unsigned char)0xF7, (0xE8 | encode));
 2380 }
 2381 
 2382 void Assembler::imull(Register dst, Register src) {
 2383   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2384   emit_int24(0x0F,
 2385              (unsigned char)0xAF,
 2386              (0xC0 | encode));
 2387 }
 2388 
 2389 void Assembler::imull(Register dst, Address src, int32_t value) {
 2390   InstructionMark im(this);
 2391   prefix(src, dst);
 2392   if (is8bit(value)) {
 2393     emit_int8((unsigned char)0x6B);
 2394     emit_operand(dst, src, 1);
 2395     emit_int8(value);
 2396   } else {
 2397     emit_int8((unsigned char)0x69);
 2398     emit_operand(dst, src, 4);
 2399     emit_int32(value);
 2400   }
 2401 }
 2402 
 2403 void Assembler::imull(Register dst, Register src, int value) {
 2404   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2405   if (is8bit(value)) {
 2406     emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
 2407   } else {
 2408     emit_int16(0x69, (0xC0 | encode));
 2409     emit_int32(value);
 2410   }
 2411 }
 2412 
 2413 void Assembler::imull(Register dst, Address src) {
 2414   InstructionMark im(this);
 2415   prefix(src, dst);
 2416   emit_int16(0x0F, (unsigned char)0xAF);
 2417   emit_operand(dst, src, 0);
 2418 }
 2419 
 2420 
 2421 void Assembler::incl(Address dst) {
 2422   // Don't use it directly. Use MacroAssembler::increment() instead.
 2423   InstructionMark im(this);
 2424   prefix(dst);
 2425   emit_int8((unsigned char)0xFF);
 2426   emit_operand(rax, dst, 0);
 2427 }
 2428 
 2429 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
 2430   InstructionMark im(this);
 2431   assert((0 <= cc) && (cc < 16), "illegal cc");
 2432   if (L.is_bound()) {
 2433     address dst = target(L);
 2434     assert(dst != nullptr, "jcc most probably wrong");
 2435 
 2436     const int short_size = 2;
 2437     const int long_size = 6;
 2438     int offs = checked_cast<int>((intptr_t)dst - (intptr_t)pc());
 2439     if (maybe_short && is8bit(offs - short_size)) {
 2440       // 0111 tttn #8-bit disp
 2441       emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2442     } else {
 2443       // 0000 1111 1000 tttn #32-bit disp
 2444       assert(is_simm32(offs - long_size),
 2445              "must be 32bit offset (call4)");
 2446       emit_int16(0x0F, (0x80 | cc));
 2447       emit_int32(offs - long_size);
 2448     }
 2449   } else {
 2450     // Note: could eliminate cond. jumps to this jump if condition
 2451     //       is the same however, seems to be rather unlikely case.
 2452     // Note: use jccb() if label to be bound is very close to get
 2453     //       an 8-bit displacement
 2454     L.add_patch_at(code(), locator());
 2455     emit_int16(0x0F, (0x80 | cc));
 2456     emit_int32(0);
 2457   }
 2458 }
 2459 
 2460 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
 2461   if (L.is_bound()) {
 2462     const int short_size = 2;
 2463     address entry = target(L);
 2464 #ifdef ASSERT
 2465     int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size));
 2466     int delta = short_branch_delta();
 2467     if (delta != 0) {
 2468       dist += (dist < 0 ? (-delta) :delta);
 2469     }
 2470     assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line);
 2471 #endif
 2472     int offs = checked_cast<int>((intptr_t)entry - (intptr_t)pc());
 2473     // 0111 tttn #8-bit disp
 2474     emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
 2475   } else {
 2476     InstructionMark im(this);
 2477     L.add_patch_at(code(), locator(), file, line);
 2478     emit_int16(0x70 | cc, 0);
 2479   }
 2480 }
 2481 
 2482 void Assembler::jmp(Address adr) {
 2483   InstructionMark im(this);
 2484   prefix(adr);
 2485   emit_int8((unsigned char)0xFF);
 2486   emit_operand(rsp, adr, 0);
 2487 }
 2488 
 2489 void Assembler::jmp(Label& L, bool maybe_short) {
 2490   if (L.is_bound()) {
 2491     address entry = target(L);
 2492     assert(entry != nullptr, "jmp most probably wrong");
 2493     InstructionMark im(this);
 2494     const int short_size = 2;
 2495     const int long_size = 5;
 2496     int offs = checked_cast<int>(entry - pc());
 2497     if (maybe_short && is8bit(offs - short_size)) {
 2498       emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
 2499     } else {
 2500       emit_int8((unsigned char)0xE9);
 2501       emit_int32(offs - long_size);
 2502     }
 2503   } else {
 2504     // By default, forward jumps are always 32-bit displacements, since
 2505     // we can't yet know where the label will be bound.  If you're sure that
 2506     // the forward jump will not run beyond 256 bytes, use jmpb to
 2507     // force an 8-bit displacement.
 2508     InstructionMark im(this);
 2509     L.add_patch_at(code(), locator());
 2510     emit_int8((unsigned char)0xE9);
 2511     emit_int32(0);
 2512   }
 2513 }
 2514 
 2515 void Assembler::jmp(Register entry) {
 2516   int encode = prefix_and_encode(entry->encoding());
 2517   emit_int16((unsigned char)0xFF, (0xE0 | encode));
 2518 }
 2519 
 2520 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
 2521   InstructionMark im(this);
 2522   emit_int8((unsigned char)0xE9);
 2523   assert(dest != nullptr, "must have a target");
 2524   intptr_t disp = dest - (pc() + sizeof(int32_t));
 2525   assert(is_simm32(disp), "must be 32bit offset (jmp)");
 2526   emit_data(checked_cast<int32_t>(disp), rspec, call32_operand);
 2527 }
 2528 
 2529 void Assembler::jmpb_0(Label& L, const char* file, int line) {
 2530   if (L.is_bound()) {
 2531     const int short_size = 2;
 2532     address entry = target(L);
 2533     assert(entry != nullptr, "jmp most probably wrong");
 2534 #ifdef ASSERT
 2535     int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size));
 2536     int delta = short_branch_delta();
 2537     if (delta != 0) {
 2538       dist += (dist < 0 ? (-delta) :delta);
 2539     }
 2540     assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line);
 2541 #endif
 2542     intptr_t offs = entry - pc();
 2543     emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
 2544   } else {
 2545     InstructionMark im(this);
 2546     L.add_patch_at(code(), locator(), file, line);
 2547     emit_int16((unsigned char)0xEB, 0);
 2548   }
 2549 }
 2550 
 2551 void Assembler::ldmxcsr( Address src) {
 2552   if (UseAVX > 0 ) {
 2553     InstructionMark im(this);
 2554     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2555     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2556     emit_int8((unsigned char)0xAE);
 2557     emit_operand(as_Register(2), src, 0);
 2558   } else {
 2559     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2560     InstructionMark im(this);
 2561     prefix(src);
 2562     emit_int16(0x0F, (unsigned char)0xAE);
 2563     emit_operand(as_Register(2), src, 0);
 2564   }
 2565 }
 2566 
 2567 void Assembler::leal(Register dst, Address src) {
 2568   InstructionMark im(this);
 2569   prefix(src, dst);
 2570   emit_int8((unsigned char)0x8D);
 2571   emit_operand(dst, src, 0);
 2572 }
 2573 
 2574 void Assembler::lfence() {
 2575   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
 2576 }
 2577 
 2578 void Assembler::lock() {
 2579   emit_int8((unsigned char)0xF0);
 2580 }
 2581 
 2582 void Assembler::size_prefix() {
 2583   emit_int8(0x66);
 2584 }
 2585 
 2586 void Assembler::lzcntl(Register dst, Register src) {
 2587   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2588   emit_int8((unsigned char)0xF3);
 2589   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 2590   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
 2591 }
 2592 
 2593 void Assembler::lzcntl(Register dst, Address src) {
 2594   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 2595   InstructionMark im(this);
 2596   emit_int8((unsigned char)0xF3);
 2597   prefix(src, dst);
 2598   emit_int16(0x0F, (unsigned char)0xBD);
 2599   emit_operand(dst, src, 0);
 2600 }
 2601 
 2602 // Emit mfence instruction
 2603 void Assembler::mfence() {
 2604   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2605   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
 2606 }
 2607 
 2608 // Emit sfence instruction
 2609 void Assembler::sfence() {
 2610   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
 2611   emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
 2612 }
 2613 
 2614 void Assembler::mov(Register dst, Register src) {
 2615   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2616 }
 2617 
 2618 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 2619   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 2620   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2621   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2622   attributes.set_rex_vex_w_reverted();
 2623   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2624   emit_int16(0x28, (0xC0 | encode));
 2625 }
 2626 
 2627 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 2628   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2629   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2630   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2631   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2632   emit_int16(0x28, (0xC0 | encode));
 2633 }
 2634 
 2635 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 2636   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 2637   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2638   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2639   emit_int16(0x16, (0xC0 | encode));
 2640 }
 2641 
 2642 void Assembler::movb(Register dst, Address src) {
 2643   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 2644   InstructionMark im(this);
 2645   prefix(src, dst, true);
 2646   emit_int8((unsigned char)0x8A);
 2647   emit_operand(dst, src, 0);
 2648 }
 2649 
 2650 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
 2651   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2652   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 2653   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2654   attributes.set_rex_vex_w_reverted();
 2655   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2656   emit_int16(0x12, 0xC0 | encode);
 2657 }
 2658 
 2659 void Assembler::movddup(XMMRegister dst, Address src) {
 2660   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
 2661   InstructionMark im(this);
 2662   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2663   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2664   attributes.set_rex_vex_w_reverted();
 2665   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2666   emit_int8(0x12);
 2667   emit_operand(dst, src, 0);
 2668 }
 2669 
 2670 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
 2671   assert(VM_Version::supports_avx(), "");
 2672   InstructionMark im(this);
 2673   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 2674   attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
 2675   attributes.set_rex_vex_w_reverted();
 2676   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2677   emit_int8(0x12);
 2678   emit_operand(dst, src, 0);
 2679 }
 2680 
 2681 void Assembler::kmovbl(KRegister dst, KRegister src) {
 2682   assert(VM_Version::supports_avx512dq(), "");
 2683   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2684   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2685   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2686 }
 2687 
 2688 void Assembler::kmovbl(KRegister dst, Register src) {
 2689   assert(VM_Version::supports_avx512dq(), "");
 2690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2691   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2692   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2693 }
 2694 
 2695 void Assembler::kmovbl(Register dst, KRegister src) {
 2696   assert(VM_Version::supports_avx512dq(), "");
 2697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2698   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2699   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2700 }
 2701 
 2702 void Assembler::kmovwl(KRegister dst, Register src) {
 2703   assert(VM_Version::supports_evex(), "");
 2704   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2705   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2706   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2707 }
 2708 
 2709 void Assembler::kmovwl(Register dst, KRegister src) {
 2710   assert(VM_Version::supports_evex(), "");
 2711   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2712   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2713   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2714 }
 2715 
 2716 void Assembler::kmovwl(KRegister dst, Address src) {
 2717   assert(VM_Version::supports_evex(), "");
 2718   InstructionMark im(this);
 2719   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2720   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2721   emit_int8((unsigned char)0x90);
 2722   emit_operand(dst, src, 0);
 2723 }
 2724 
 2725 void Assembler::kmovwl(Address dst, KRegister src) {
 2726   assert(VM_Version::supports_evex(), "");
 2727   InstructionMark im(this);
 2728   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2729   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2730   emit_int8((unsigned char)0x91);
 2731   emit_operand(src, dst, 0);
 2732 }
 2733 
 2734 void Assembler::kmovwl(KRegister dst, KRegister src) {
 2735   assert(VM_Version::supports_evex(), "");
 2736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2737   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2738   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2739 }
 2740 
 2741 void Assembler::kmovdl(KRegister dst, Register src) {
 2742   assert(VM_Version::supports_avx512bw(), "");
 2743   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2744   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2745   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2746 }
 2747 
 2748 void Assembler::kmovdl(Register dst, KRegister src) {
 2749   assert(VM_Version::supports_avx512bw(), "");
 2750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2751   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2752   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2753 }
 2754 
 2755 void Assembler::kmovql(KRegister dst, KRegister src) {
 2756   assert(VM_Version::supports_avx512bw(), "");
 2757   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2758   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2759   emit_int16((unsigned char)0x90, (0xC0 | encode));
 2760 }
 2761 
 2762 void Assembler::kmovql(KRegister dst, Address src) {
 2763   assert(VM_Version::supports_avx512bw(), "");
 2764   InstructionMark im(this);
 2765   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2766   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2767   emit_int8((unsigned char)0x90);
 2768   emit_operand(dst, src, 0);
 2769 }
 2770 
 2771 void Assembler::kmovql(Address dst, KRegister src) {
 2772   assert(VM_Version::supports_avx512bw(), "");
 2773   InstructionMark im(this);
 2774   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2775   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2776   emit_int8((unsigned char)0x91);
 2777   emit_operand(src, dst, 0);
 2778 }
 2779 
 2780 void Assembler::kmovql(KRegister dst, Register src) {
 2781   assert(VM_Version::supports_avx512bw(), "");
 2782   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2783   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2784   emit_int16((unsigned char)0x92, (0xC0 | encode));
 2785 }
 2786 
 2787 void Assembler::kmovql(Register dst, KRegister src) {
 2788   assert(VM_Version::supports_avx512bw(), "");
 2789   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2790   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 2791   emit_int16((unsigned char)0x93, (0xC0 | encode));
 2792 }
 2793 
 2794 void Assembler::knotwl(KRegister dst, KRegister src) {
 2795   assert(VM_Version::supports_evex(), "");
 2796   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2797   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2798   emit_int16(0x44, (0xC0 | encode));
 2799 }
 2800 
 2801 void Assembler::knotbl(KRegister dst, KRegister src) {
 2802   assert(VM_Version::supports_evex(), "");
 2803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2804   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2805   emit_int16(0x44, (0xC0 | encode));
 2806 }
 2807 
 2808 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
 2809   assert(VM_Version::supports_avx512dq(), "");
 2810   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2811   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2812   emit_int16(0x45, (0xC0 | encode));
 2813 }
 2814 
 2815 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
 2816   assert(VM_Version::supports_evex(), "");
 2817   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2818   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2819   emit_int16(0x45, (0xC0 | encode));
 2820 }
 2821 
 2822 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
 2823   assert(VM_Version::supports_avx512bw(), "");
 2824   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2825   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2826   emit_int16(0x45, (0xC0 | encode));
 2827 }
 2828 
 2829 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
 2830   assert(VM_Version::supports_avx512bw(), "");
 2831   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2832   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2833   emit_int16(0x45, (0xC0 | encode));
 2834 }
 2835 
 2836 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
 2837   assert(VM_Version::supports_avx512dq(), "");
 2838   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2839   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2840   emit_int16(0x47, (0xC0 | encode));
 2841 }
 2842 
 2843 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
 2844   assert(VM_Version::supports_evex(), "");
 2845   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2846   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2847   emit_int16(0x47, (0xC0 | encode));
 2848 }
 2849 
 2850 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
 2851   assert(VM_Version::supports_avx512bw(), "");
 2852   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2853   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2854   emit_int16(0x47, (0xC0 | encode));
 2855 }
 2856 
 2857 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
 2858   assert(VM_Version::supports_avx512bw(), "");
 2859   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2860   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2861   emit_int16(0x47, (0xC0 | encode));
 2862 }
 2863 
 2864 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
 2865   assert(VM_Version::supports_avx512dq(), "");
 2866   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2867   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2868   emit_int16(0x41, (0xC0 | encode));
 2869 }
 2870 
 2871 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
 2872   assert(VM_Version::supports_evex(), "");
 2873   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2874   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2875   emit_int16(0x41, (0xC0 | encode));
 2876 }
 2877 
 2878 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
 2879   assert(VM_Version::supports_avx512bw(), "");
 2880   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2881   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2882   emit_int16(0x41, (0xC0 | encode));
 2883 }
 2884 
 2885 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
 2886   assert(VM_Version::supports_avx512bw(), "");
 2887   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2888   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2889   emit_int16(0x41, (0xC0 | encode));
 2890 }
 2891 
 2892 void Assembler::knotdl(KRegister dst, KRegister src) {
 2893   assert(VM_Version::supports_avx512bw(), "");
 2894   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2895   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2896   emit_int16(0x44, (0xC0 | encode));
 2897 }
 2898 
 2899 void Assembler::knotql(KRegister dst, KRegister src) {
 2900   assert(VM_Version::supports_avx512bw(), "");
 2901   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2902   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2903   emit_int16(0x44, (0xC0 | encode));
 2904 }
 2905 
 2906 // This instruction produces ZF or CF flags
 2907 void Assembler::kortestbl(KRegister src1, KRegister src2) {
 2908   assert(VM_Version::supports_avx512dq(), "");
 2909   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2910   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2911   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2912 }
 2913 
 2914 // This instruction produces ZF or CF flags
 2915 void Assembler::kortestwl(KRegister src1, KRegister src2) {
 2916   assert(VM_Version::supports_evex(), "");
 2917   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2918   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2919   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2920 }
 2921 
 2922 // This instruction produces ZF or CF flags
 2923 void Assembler::kortestdl(KRegister src1, KRegister src2) {
 2924   assert(VM_Version::supports_avx512bw(), "");
 2925   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2926   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2927   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2928 }
 2929 
 2930 // This instruction produces ZF or CF flags
 2931 void Assembler::kortestql(KRegister src1, KRegister src2) {
 2932   assert(VM_Version::supports_avx512bw(), "");
 2933   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2934   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2935   emit_int16((unsigned char)0x98, (0xC0 | encode));
 2936 }
 2937 
 2938 // This instruction produces ZF or CF flags
 2939 void Assembler::ktestql(KRegister src1, KRegister src2) {
 2940   assert(VM_Version::supports_avx512bw(), "");
 2941   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2942   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2943   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2944 }
 2945 
 2946 void Assembler::ktestdl(KRegister src1, KRegister src2) {
 2947   assert(VM_Version::supports_avx512bw(), "");
 2948   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2949   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2950   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2951 }
 2952 
 2953 void Assembler::ktestwl(KRegister src1, KRegister src2) {
 2954   assert(VM_Version::supports_avx512dq(), "");
 2955   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2956   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2957   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2958 }
 2959 
 2960 void Assembler::ktestbl(KRegister src1, KRegister src2) {
 2961   assert(VM_Version::supports_avx512dq(), "");
 2962   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2963   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2964   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2965 }
 2966 
 2967 void Assembler::ktestq(KRegister src1, KRegister src2) {
 2968   assert(VM_Version::supports_avx512bw(), "");
 2969   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2970   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 2971   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2972 }
 2973 
 2974 void Assembler::ktestd(KRegister src1, KRegister src2) {
 2975   assert(VM_Version::supports_avx512bw(), "");
 2976   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2977   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2978   emit_int16((unsigned char)0x99, (0xC0 | encode));
 2979 }
 2980 
 2981 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
 2982   assert(VM_Version::supports_avx512dq(), "");
 2983   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2984   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 2985   emit_int16(0x46, (0xC0 | encode));
 2986 }
 2987 
 2988 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
 2989   assert(VM_Version::supports_avx512dq(), "");
 2990   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2991   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 2992   emit_int16(0x32, (0xC0 | encode));
 2993   emit_int8(imm8);
 2994 }
 2995 
 2996 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
 2997   assert(VM_Version::supports_avx512bw(), "");
 2998   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 2999   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3000   emit_int16(0x33, (0xC0 | encode));
 3001   emit_int8(imm8);
 3002 }
 3003 
 3004 
 3005 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
 3006   assert(VM_Version::supports_avx512dq(), "");
 3007   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3008   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3009   emit_int16(0x30, (0xC0 | encode));
 3010 }
 3011 
 3012 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
 3013   assert(VM_Version::supports_evex(), "");
 3014   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3015   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3016   emit_int16(0x30, (0xC0 | encode));
 3017   emit_int8(imm8);
 3018 }
 3019 
 3020 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
 3021   assert(VM_Version::supports_avx512bw(), "");
 3022   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3023   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3024   emit_int16(0x31, (0xC0 | encode));
 3025   emit_int8(imm8);
 3026 }
 3027 
 3028 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
 3029   assert(VM_Version::supports_avx512bw(), "");
 3030   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3031   int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 3032   emit_int16(0x31, (0xC0 | encode));
 3033   emit_int8(imm8);
 3034 }
 3035 
 3036 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
 3037   assert(VM_Version::supports_avx512bw(), "");
 3038   InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3039   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3040   emit_int16(0x4B, (0xC0 | encode));
 3041 }
 3042 
 3043 void Assembler::movb(Address dst, int imm8) {
 3044   InstructionMark im(this);
 3045    prefix(dst);
 3046   emit_int8((unsigned char)0xC6);
 3047   emit_operand(rax, dst, 1);
 3048   emit_int8(imm8);
 3049 }
 3050 
 3051 
 3052 void Assembler::movb(Address dst, Register src) {
 3053   assert(src->has_byte_register(), "must have byte register");
 3054   InstructionMark im(this);
 3055   prefix(dst, src, true);
 3056   emit_int8((unsigned char)0x88);
 3057   emit_operand(src, dst, 0);
 3058 }
 3059 
 3060 void Assembler::movdl(XMMRegister dst, Register src) {
 3061   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3063   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3064   emit_int16(0x6E, (0xC0 | encode));
 3065 }
 3066 
 3067 void Assembler::movdl(Register dst, XMMRegister src) {
 3068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3070   // swap src/dst to get correct prefix
 3071   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3072   emit_int16(0x7E, (0xC0 | encode));
 3073 }
 3074 
 3075 void Assembler::movdl(XMMRegister dst, Address src) {
 3076   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3077   InstructionMark im(this);
 3078   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3079   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3080   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3081   emit_int8(0x6E);
 3082   emit_operand(dst, src, 0);
 3083 }
 3084 
 3085 void Assembler::movdl(Address dst, XMMRegister src) {
 3086   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3087   InstructionMark im(this);
 3088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3089   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3090   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3091   emit_int8(0x7E);
 3092   emit_operand(src, dst, 0);
 3093 }
 3094 
 3095 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 3096   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3098   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3099   emit_int16(0x6F, (0xC0 | encode));
 3100 }
 3101 
 3102 void Assembler::movdqa(XMMRegister dst, Address src) {
 3103   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3104   InstructionMark im(this);
 3105   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3106   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3107   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3108   emit_int8(0x6F);
 3109   emit_operand(dst, src, 0);
 3110 }
 3111 
 3112 void Assembler::movdqu(XMMRegister dst, Address src) {
 3113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3114   InstructionMark im(this);
 3115   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3116   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3117   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3118   emit_int8(0x6F);
 3119   emit_operand(dst, src, 0);
 3120 }
 3121 
 3122 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 3123   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3124   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3125   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3126   emit_int16(0x6F, (0xC0 | encode));
 3127 }
 3128 
 3129 void Assembler::movdqu(Address dst, XMMRegister src) {
 3130   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3131   InstructionMark im(this);
 3132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3133   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3134   attributes.reset_is_clear_context();
 3135   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3136   emit_int8(0x7F);
 3137   emit_operand(src, dst, 0);
 3138 }
 3139 
 3140 // Move Unaligned 256bit Vector
 3141 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 3142   assert(UseAVX > 0, "");
 3143   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3144   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3145   emit_int16(0x6F, (0xC0 | encode));
 3146 }
 3147 
 3148 void Assembler::vmovdqu(XMMRegister dst, Address src) {
 3149   assert(UseAVX > 0, "");
 3150   InstructionMark im(this);
 3151   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3152   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3153   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3154   emit_int8(0x6F);
 3155   emit_operand(dst, src, 0);
 3156 }
 3157 
 3158 void Assembler::vmovdqu(Address dst, XMMRegister src) {
 3159   assert(UseAVX > 0, "");
 3160   InstructionMark im(this);
 3161   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3162   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3163   attributes.reset_is_clear_context();
 3164   // swap src<->dst for encoding
 3165   assert(src != xnoreg, "sanity");
 3166   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3167   emit_int8(0x7F);
 3168   emit_operand(src, dst, 0);
 3169 }
 3170 
 3171 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3172   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3173   InstructionMark im(this);
 3174   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3175   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3176   emit_int8((unsigned char)0x8C);
 3177   emit_operand(dst, src, 0);
 3178 }
 3179 
 3180 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
 3181   assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
 3182   InstructionMark im(this);
 3183   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
 3184   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3185   emit_int8((unsigned char)0x8C);
 3186   emit_operand(dst, src, 0);
 3187 }
 3188 
 3189 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3190   assert(UseAVX > 0, "requires some form of AVX");
 3191   InstructionMark im(this);
 3192   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3193   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3194   emit_int8(0x2C);
 3195   emit_operand(dst, src, 0);
 3196 }
 3197 
 3198 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
 3199   assert(UseAVX > 0, "requires some form of AVX");
 3200   InstructionMark im(this);
 3201   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3202   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3203   emit_int8(0x2D);
 3204   emit_operand(dst, src, 0);
 3205 }
 3206 
 3207 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3208   assert(UseAVX > 0, "");
 3209   InstructionMark im(this);
 3210   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3211   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3212   emit_int8(0x2E);
 3213   emit_operand(src, dst, 0);
 3214 }
 3215 
 3216 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
 3217   assert(UseAVX > 0, "");
 3218   InstructionMark im(this);
 3219   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 3220   vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 3221   emit_int8(0x2F);
 3222   emit_operand(src, dst, 0);
 3223 }
 3224 
 3225 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
 3226 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3227   assert(VM_Version::supports_avx512vlbw(), "");
 3228   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3229   attributes.set_embedded_opmask_register_specifier(mask);
 3230   attributes.set_is_evex_instruction();
 3231   if (merge) {
 3232     attributes.reset_is_clear_context();
 3233   }
 3234   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3235   emit_int16(0x6F, (0xC0 | encode));
 3236 }
 3237 
 3238 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
 3239   // Unmasked instruction
 3240   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3241 }
 3242 
 3243 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3244   assert(VM_Version::supports_avx512vlbw(), "");
 3245   InstructionMark im(this);
 3246   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3247   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3248   attributes.set_embedded_opmask_register_specifier(mask);
 3249   attributes.set_is_evex_instruction();
 3250   if (merge) {
 3251     attributes.reset_is_clear_context();
 3252   }
 3253   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3254   emit_int8(0x6F);
 3255   emit_operand(dst, src, 0);
 3256 }
 3257 
 3258 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
 3259   // Unmasked instruction
 3260   evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
 3261 }
 3262 
 3263 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3264   assert(VM_Version::supports_avx512vlbw(), "");
 3265   assert(src != xnoreg, "sanity");
 3266   InstructionMark im(this);
 3267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3268   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3269   attributes.set_embedded_opmask_register_specifier(mask);
 3270   attributes.set_is_evex_instruction();
 3271   if (merge) {
 3272     attributes.reset_is_clear_context();
 3273   }
 3274   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3275   emit_int8(0x7F);
 3276   emit_operand(src, dst, 0);
 3277 }
 3278 
 3279 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
 3280   // Unmasked instruction
 3281   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3282 }
 3283 
 3284 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3285   assert(VM_Version::supports_avx512vlbw(), "");
 3286   InstructionMark im(this);
 3287   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3288   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3289   attributes.set_embedded_opmask_register_specifier(mask);
 3290   attributes.set_is_evex_instruction();
 3291   if (merge) {
 3292     attributes.reset_is_clear_context();
 3293   }
 3294   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3295   emit_int8(0x6F);
 3296   emit_operand(dst, src, 0);
 3297 }
 3298 
 3299 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
 3300   // Unmasked instruction
 3301   evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
 3302 }
 3303 
 3304 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3305   assert(VM_Version::supports_avx512vlbw(), "");
 3306   assert(src != xnoreg, "sanity");
 3307   InstructionMark im(this);
 3308   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 3309   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3310   attributes.set_embedded_opmask_register_specifier(mask);
 3311   attributes.set_is_evex_instruction();
 3312   if (merge) {
 3313     attributes.reset_is_clear_context();
 3314   }
 3315   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3316   emit_int8(0x7F);
 3317   emit_operand(src, dst, 0);
 3318 }
 3319 
 3320 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
 3321   // Unmasked instruction
 3322   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3323 }
 3324 
 3325 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3326   assert(VM_Version::supports_evex(), "");
 3327   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3328   attributes.set_embedded_opmask_register_specifier(mask);
 3329   attributes.set_is_evex_instruction();
 3330   if (merge) {
 3331     attributes.reset_is_clear_context();
 3332   }
 3333   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3334   emit_int16(0x6F, (0xC0 | encode));
 3335 }
 3336 
 3337 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
 3338   // Unmasked instruction
 3339   evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
 3340 }
 3341 
 3342 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3343   assert(VM_Version::supports_evex(), "");
 3344   InstructionMark im(this);
 3345   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
 3346   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3347   attributes.set_embedded_opmask_register_specifier(mask);
 3348   attributes.set_is_evex_instruction();
 3349   if (merge) {
 3350     attributes.reset_is_clear_context();
 3351   }
 3352   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3353   emit_int8(0x6F);
 3354   emit_operand(dst, src, 0);
 3355 }
 3356 
 3357 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
 3358   // Unmasked isntruction
 3359   evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
 3360 }
 3361 
 3362 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3363   assert(VM_Version::supports_evex(), "");
 3364   assert(src != xnoreg, "sanity");
 3365   InstructionMark im(this);
 3366   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3367   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3368   attributes.set_embedded_opmask_register_specifier(mask);
 3369   attributes.set_is_evex_instruction();
 3370   if (merge) {
 3371     attributes.reset_is_clear_context();
 3372   }
 3373   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3374   emit_int8(0x7F);
 3375   emit_operand(src, dst, 0);
 3376 }
 3377 
 3378 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
 3379   // Unmasked instruction
 3380   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3381 }
 3382 
 3383 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3384   assert(VM_Version::supports_evex(), "");
 3385   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3386   attributes.set_embedded_opmask_register_specifier(mask);
 3387   attributes.set_is_evex_instruction();
 3388   if (merge) {
 3389     attributes.reset_is_clear_context();
 3390   }
 3391   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3392   emit_int16(0x6F, (0xC0 | encode));
 3393 }
 3394 
 3395 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
 3396   // Unmasked instruction
 3397   evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
 3398 }
 3399 
 3400 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 3401   assert(VM_Version::supports_evex(), "");
 3402   InstructionMark im(this);
 3403   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3404   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3405   attributes.set_embedded_opmask_register_specifier(mask);
 3406   attributes.set_is_evex_instruction();
 3407   if (merge) {
 3408     attributes.reset_is_clear_context();
 3409   }
 3410   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3411   emit_int8(0x6F);
 3412   emit_operand(dst, src, 0);
 3413 }
 3414 
 3415 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
 3416   // Unmasked instruction
 3417   evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
 3418 }
 3419 
 3420 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 3421   assert(VM_Version::supports_evex(), "");
 3422   assert(src != xnoreg, "sanity");
 3423   InstructionMark im(this);
 3424   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 3425   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 3426   attributes.set_embedded_opmask_register_specifier(mask);
 3427   if (merge) {
 3428     attributes.reset_is_clear_context();
 3429   }
 3430   attributes.set_is_evex_instruction();
 3431   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3432   emit_int8(0x7F);
 3433   emit_operand(src, dst, 0);
 3434 }
 3435 
 3436 // Uses zero extension on 64bit
 3437 
 3438 void Assembler::movl(Register dst, int32_t imm32) {
 3439   int encode = prefix_and_encode(dst->encoding());
 3440   emit_int8(0xB8 | encode);
 3441   emit_int32(imm32);
 3442 }
 3443 
 3444 void Assembler::movl(Register dst, Register src) {
 3445   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3446   emit_int16((unsigned char)0x8B, (0xC0 | encode));
 3447 }
 3448 
 3449 void Assembler::movl(Register dst, Address src) {
 3450   InstructionMark im(this);
 3451   prefix(src, dst);
 3452   emit_int8((unsigned char)0x8B);
 3453   emit_operand(dst, src, 0);
 3454 }
 3455 
 3456 void Assembler::movl(Address dst, int32_t imm32) {
 3457   InstructionMark im(this);
 3458   prefix(dst);
 3459   emit_int8((unsigned char)0xC7);
 3460   emit_operand(rax, dst, 4);
 3461   emit_int32(imm32);
 3462 }
 3463 
 3464 void Assembler::movl(Address dst, Register src) {
 3465   InstructionMark im(this);
 3466   prefix(dst, src);
 3467   emit_int8((unsigned char)0x89);
 3468   emit_operand(src, dst, 0);
 3469 }
 3470 
 3471 // New cpus require to use movsd and movss to avoid partial register stall
 3472 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 3473 // The selection is done in MacroAssembler::movdbl() and movflt().
 3474 void Assembler::movlpd(XMMRegister dst, Address src) {
 3475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3476   InstructionMark im(this);
 3477   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3478   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3479   attributes.set_rex_vex_w_reverted();
 3480   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3481   emit_int8(0x12);
 3482   emit_operand(dst, src, 0);
 3483 }
 3484 
 3485 void Assembler::movq(XMMRegister dst, Address src) {
 3486   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3487   InstructionMark im(this);
 3488   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3489   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3490   attributes.set_rex_vex_w_reverted();
 3491   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3492   emit_int8(0x7E);
 3493   emit_operand(dst, src, 0);
 3494 }
 3495 
 3496 void Assembler::movq(Address dst, XMMRegister src) {
 3497   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3498   InstructionMark im(this);
 3499   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3500   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3501   attributes.set_rex_vex_w_reverted();
 3502   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3503   emit_int8((unsigned char)0xD6);
 3504   emit_operand(src, dst, 0);
 3505 }
 3506 
 3507 void Assembler::movq(XMMRegister dst, XMMRegister src) {
 3508   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3509   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3510   attributes.set_rex_vex_w_reverted();
 3511   int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3512   emit_int16((unsigned char)0xD6, (0xC0 | encode));
 3513 }
 3514 
 3515 void Assembler::movq(Register dst, XMMRegister src) {
 3516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3517   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3518   // swap src/dst to get correct prefix
 3519   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3520   emit_int16(0x7E, (0xC0 | encode));
 3521 }
 3522 
 3523 void Assembler::movq(XMMRegister dst, Register src) {
 3524   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3525   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3526   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 3527   emit_int16(0x6E, (0xC0 | encode));
 3528 }
 3529 
 3530 void Assembler::movsbl(Register dst, Address src) { // movsxb
 3531   InstructionMark im(this);
 3532   prefix(src, dst);
 3533   emit_int16(0x0F, (unsigned char)0xBE);
 3534   emit_operand(dst, src, 0);
 3535 }
 3536 
 3537 void Assembler::movsbl(Register dst, Register src) { // movsxb
 3538   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3539   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3540   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
 3541 }
 3542 
 3543 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 3544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3545   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3546   attributes.set_rex_vex_w_reverted();
 3547   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3548   emit_int16(0x10, (0xC0 | encode));
 3549 }
 3550 
 3551 void Assembler::movsd(XMMRegister dst, Address src) {
 3552   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3553   InstructionMark im(this);
 3554   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3555   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3556   attributes.set_rex_vex_w_reverted();
 3557   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3558   emit_int8(0x10);
 3559   emit_operand(dst, src, 0);
 3560 }
 3561 
 3562 void Assembler::movsd(Address dst, XMMRegister src) {
 3563   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3564   InstructionMark im(this);
 3565   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3566   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3567   attributes.reset_is_clear_context();
 3568   attributes.set_rex_vex_w_reverted();
 3569   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3570   emit_int8(0x11);
 3571   emit_operand(src, dst, 0);
 3572 }
 3573 
 3574 void Assembler::vmovsd(XMMRegister dst, XMMRegister src, XMMRegister src2) {
 3575   assert(UseAVX > 0, "Requires some form of AVX");
 3576   InstructionMark im(this);
 3577   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3578   int encode = vex_prefix_and_encode(src2->encoding(), src->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3579   emit_int16(0x11, (0xC0 | encode));
 3580 }
 3581 
 3582 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 3583   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3584   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3585   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3586   emit_int16(0x10, (0xC0 | encode));
 3587 }
 3588 
 3589 void Assembler::movss(XMMRegister dst, Address src) {
 3590   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3591   InstructionMark im(this);
 3592   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3593   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3594   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3595   emit_int8(0x10);
 3596   emit_operand(dst, src, 0);
 3597 }
 3598 
 3599 void Assembler::movss(Address dst, XMMRegister src) {
 3600   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3601   InstructionMark im(this);
 3602   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3603   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3604   attributes.reset_is_clear_context();
 3605   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3606   emit_int8(0x11);
 3607   emit_operand(src, dst, 0);
 3608 }
 3609 
 3610 void Assembler::movswl(Register dst, Address src) { // movsxw
 3611   InstructionMark im(this);
 3612   prefix(src, dst);
 3613   emit_int16(0x0F, (unsigned char)0xBF);
 3614   emit_operand(dst, src, 0);
 3615 }
 3616 
 3617 void Assembler::movswl(Register dst, Register src) { // movsxw
 3618   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3619   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
 3620 }
 3621 
 3622 void Assembler::movups(XMMRegister dst, Address src) {
 3623   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3624   InstructionMark im(this);
 3625   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3626   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3627   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3628   emit_int8(0x10);
 3629   emit_operand(dst, src, 0);
 3630 }
 3631 
 3632 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
 3633   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3634   InstructionMark im(this);
 3635   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3636   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3637   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3638   emit_int8(0x10);
 3639   emit_operand(dst, src, 0);
 3640 }
 3641 
 3642 void Assembler::movups(Address dst, XMMRegister src) {
 3643   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3644   InstructionMark im(this);
 3645   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3646   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3647   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3648   emit_int8(0x11);
 3649   emit_operand(src, dst, 0);
 3650 }
 3651 
 3652 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
 3653   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 3654   InstructionMark im(this);
 3655   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 3656   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
 3657   simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 3658   emit_int8(0x11);
 3659   emit_operand(src, dst, 0);
 3660 }
 3661 
 3662 void Assembler::movw(Address dst, int imm16) {
 3663   InstructionMark im(this);
 3664 
 3665   emit_int8(0x66); // switch to 16-bit mode
 3666   prefix(dst);
 3667   emit_int8((unsigned char)0xC7);
 3668   emit_operand(rax, dst, 2);
 3669   emit_int16(imm16);
 3670 }
 3671 
 3672 void Assembler::movw(Register dst, Address src) {
 3673   InstructionMark im(this);
 3674   emit_int8(0x66);
 3675   prefix(src, dst);
 3676   emit_int8((unsigned char)0x8B);
 3677   emit_operand(dst, src, 0);
 3678 }
 3679 
 3680 void Assembler::movw(Address dst, Register src) {
 3681   InstructionMark im(this);
 3682   emit_int8(0x66);
 3683   prefix(dst, src);
 3684   emit_int8((unsigned char)0x89);
 3685   emit_operand(src, dst, 0);
 3686 }
 3687 
 3688 void Assembler::movzbl(Register dst, Address src) { // movzxb
 3689   InstructionMark im(this);
 3690   prefix(src, dst);
 3691   emit_int16(0x0F, (unsigned char)0xB6);
 3692   emit_operand(dst, src, 0);
 3693 }
 3694 
 3695 void Assembler::movzbl(Register dst, Register src) { // movzxb
 3696   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
 3697   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
 3698   emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
 3699 }
 3700 
 3701 void Assembler::movzwl(Register dst, Address src) { // movzxw
 3702   InstructionMark im(this);
 3703   prefix(src, dst);
 3704   emit_int16(0x0F, (unsigned char)0xB7);
 3705   emit_operand(dst, src, 0);
 3706 }
 3707 
 3708 void Assembler::movzwl(Register dst, Register src) { // movzxw
 3709   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 3710   emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
 3711 }
 3712 
 3713 void Assembler::mull(Address src) {
 3714   InstructionMark im(this);
 3715   prefix(src);
 3716   emit_int8((unsigned char)0xF7);
 3717   emit_operand(rsp, src, 0);
 3718 }
 3719 
 3720 void Assembler::mull(Register src) {
 3721   int encode = prefix_and_encode(src->encoding());
 3722   emit_int16((unsigned char)0xF7, (0xE0 | encode));
 3723 }
 3724 
 3725 void Assembler::mulsd(XMMRegister dst, Address src) {
 3726   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3727   InstructionMark im(this);
 3728   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3729   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 3730   attributes.set_rex_vex_w_reverted();
 3731   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3732   emit_int8(0x59);
 3733   emit_operand(dst, src, 0);
 3734 }
 3735 
 3736 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 3737   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 3738   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3739   attributes.set_rex_vex_w_reverted();
 3740   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 3741   emit_int16(0x59, (0xC0 | encode));
 3742 }
 3743 
 3744 void Assembler::mulss(XMMRegister dst, Address src) {
 3745   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3746   InstructionMark im(this);
 3747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3748   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 3749   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3750   emit_int8(0x59);
 3751   emit_operand(dst, src, 0);
 3752 }
 3753 
 3754 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 3755   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 3756   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 3757   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 3758   emit_int16(0x59, (0xC0 | encode));
 3759 }
 3760 
 3761 void Assembler::negl(Register dst) {
 3762   int encode = prefix_and_encode(dst->encoding());
 3763   emit_int16((unsigned char)0xF7, (0xD8 | encode));
 3764 }
 3765 
 3766 void Assembler::negl(Address dst) {
 3767   InstructionMark im(this);
 3768   prefix(dst);
 3769   emit_int8((unsigned char)0xF7);
 3770   emit_operand(as_Register(3), dst, 0);
 3771 }
 3772 
 3773 void Assembler::nop(int i) {
 3774 #ifdef ASSERT
 3775   assert(i > 0, " ");
 3776   // The fancy nops aren't currently recognized by debuggers making it a
 3777   // pain to disassemble code while debugging. If asserts are on clearly
 3778   // speed is not an issue so simply use the single byte traditional nop
 3779   // to do alignment.
 3780 
 3781   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
 3782   return;
 3783 
 3784 #endif // ASSERT
 3785 
 3786   if (UseAddressNop && VM_Version::is_intel()) {
 3787     //
 3788     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
 3789     //  1: 0x90
 3790     //  2: 0x66 0x90
 3791     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3792     //  4: 0x0F 0x1F 0x40 0x00
 3793     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3794     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3795     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3796     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3797     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3798     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3799     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3800 
 3801     // The rest coding is Intel specific - don't use consecutive address nops
 3802 
 3803     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3804     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3805     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3806     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3807 
 3808     while(i >= 15) {
 3809       // For Intel don't generate consecutive address nops (mix with regular nops)
 3810       i -= 15;
 3811       emit_int24(0x66, 0x66, 0x66);
 3812       addr_nop_8();
 3813       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3814     }
 3815     switch (i) {
 3816       case 14:
 3817         emit_int8(0x66); // size prefix
 3818       case 13:
 3819         emit_int8(0x66); // size prefix
 3820       case 12:
 3821         addr_nop_8();
 3822         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3823         break;
 3824       case 11:
 3825         emit_int8(0x66); // size prefix
 3826       case 10:
 3827         emit_int8(0x66); // size prefix
 3828       case 9:
 3829         emit_int8(0x66); // size prefix
 3830       case 8:
 3831         addr_nop_8();
 3832         break;
 3833       case 7:
 3834         addr_nop_7();
 3835         break;
 3836       case 6:
 3837         emit_int8(0x66); // size prefix
 3838       case 5:
 3839         addr_nop_5();
 3840         break;
 3841       case 4:
 3842         addr_nop_4();
 3843         break;
 3844       case 3:
 3845         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3846         emit_int8(0x66); // size prefix
 3847       case 2:
 3848         emit_int8(0x66); // size prefix
 3849       case 1:
 3850         emit_int8((unsigned char)0x90);
 3851                          // nop
 3852         break;
 3853       default:
 3854         assert(i == 0, " ");
 3855     }
 3856     return;
 3857   }
 3858   if (UseAddressNop && VM_Version::is_amd_family()) {
 3859     //
 3860     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
 3861     //  1: 0x90
 3862     //  2: 0x66 0x90
 3863     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3864     //  4: 0x0F 0x1F 0x40 0x00
 3865     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3866     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3867     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3868     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3869     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3870     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3871     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3872 
 3873     // The rest coding is AMD specific - use consecutive address nops
 3874 
 3875     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3876     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
 3877     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3878     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3879     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3880     //     Size prefixes (0x66) are added for larger sizes
 3881 
 3882     while(i >= 22) {
 3883       i -= 11;
 3884       emit_int24(0x66, 0x66, 0x66);
 3885       addr_nop_8();
 3886     }
 3887     // Generate first nop for size between 21-12
 3888     switch (i) {
 3889       case 21:
 3890         i -= 1;
 3891         emit_int8(0x66); // size prefix
 3892       case 20:
 3893       case 19:
 3894         i -= 1;
 3895         emit_int8(0x66); // size prefix
 3896       case 18:
 3897       case 17:
 3898         i -= 1;
 3899         emit_int8(0x66); // size prefix
 3900       case 16:
 3901       case 15:
 3902         i -= 8;
 3903         addr_nop_8();
 3904         break;
 3905       case 14:
 3906       case 13:
 3907         i -= 7;
 3908         addr_nop_7();
 3909         break;
 3910       case 12:
 3911         i -= 6;
 3912         emit_int8(0x66); // size prefix
 3913         addr_nop_5();
 3914         break;
 3915       default:
 3916         assert(i < 12, " ");
 3917     }
 3918 
 3919     // Generate second nop for size between 11-1
 3920     switch (i) {
 3921       case 11:
 3922         emit_int8(0x66); // size prefix
 3923       case 10:
 3924         emit_int8(0x66); // size prefix
 3925       case 9:
 3926         emit_int8(0x66); // size prefix
 3927       case 8:
 3928         addr_nop_8();
 3929         break;
 3930       case 7:
 3931         addr_nop_7();
 3932         break;
 3933       case 6:
 3934         emit_int8(0x66); // size prefix
 3935       case 5:
 3936         addr_nop_5();
 3937         break;
 3938       case 4:
 3939         addr_nop_4();
 3940         break;
 3941       case 3:
 3942         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 3943         emit_int8(0x66); // size prefix
 3944       case 2:
 3945         emit_int8(0x66); // size prefix
 3946       case 1:
 3947         emit_int8((unsigned char)0x90);
 3948                          // nop
 3949         break;
 3950       default:
 3951         assert(i == 0, " ");
 3952     }
 3953     return;
 3954   }
 3955 
 3956   if (UseAddressNop && VM_Version::is_zx()) {
 3957     //
 3958     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
 3959     //  1: 0x90
 3960     //  2: 0x66 0x90
 3961     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
 3962     //  4: 0x0F 0x1F 0x40 0x00
 3963     //  5: 0x0F 0x1F 0x44 0x00 0x00
 3964     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
 3965     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
 3966     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3967     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3968     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3969     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
 3970 
 3971     // The rest coding is ZX specific - don't use consecutive address nops
 3972 
 3973     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3974     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3975     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3976     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
 3977 
 3978     while (i >= 15) {
 3979       // For ZX don't generate consecutive address nops (mix with regular nops)
 3980       i -= 15;
 3981       emit_int24(0x66, 0x66, 0x66);
 3982       addr_nop_8();
 3983       emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3984     }
 3985     switch (i) {
 3986       case 14:
 3987         emit_int8(0x66); // size prefix
 3988       case 13:
 3989         emit_int8(0x66); // size prefix
 3990       case 12:
 3991         addr_nop_8();
 3992         emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 3993         break;
 3994       case 11:
 3995         emit_int8(0x66); // size prefix
 3996       case 10:
 3997         emit_int8(0x66); // size prefix
 3998       case 9:
 3999         emit_int8(0x66); // size prefix
 4000       case 8:
 4001         addr_nop_8();
 4002         break;
 4003       case 7:
 4004         addr_nop_7();
 4005         break;
 4006       case 6:
 4007         emit_int8(0x66); // size prefix
 4008       case 5:
 4009         addr_nop_5();
 4010         break;
 4011       case 4:
 4012         addr_nop_4();
 4013         break;
 4014       case 3:
 4015         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
 4016         emit_int8(0x66); // size prefix
 4017       case 2:
 4018         emit_int8(0x66); // size prefix
 4019       case 1:
 4020         emit_int8((unsigned char)0x90);
 4021                          // nop
 4022         break;
 4023       default:
 4024         assert(i == 0, " ");
 4025     }
 4026     return;
 4027   }
 4028 
 4029   // Using nops with size prefixes "0x66 0x90".
 4030   // From AMD Optimization Guide:
 4031   //  1: 0x90
 4032   //  2: 0x66 0x90
 4033   //  3: 0x66 0x66 0x90
 4034   //  4: 0x66 0x66 0x66 0x90
 4035   //  5: 0x66 0x66 0x90 0x66 0x90
 4036   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
 4037   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
 4038   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
 4039   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 4040   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
 4041   //
 4042   while (i > 12) {
 4043     i -= 4;
 4044     emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
 4045   }
 4046   // 1 - 12 nops
 4047   if (i > 8) {
 4048     if (i > 9) {
 4049       i -= 1;
 4050       emit_int8(0x66);
 4051     }
 4052     i -= 3;
 4053     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4054   }
 4055   // 1 - 8 nops
 4056   if (i > 4) {
 4057     if (i > 6) {
 4058       i -= 1;
 4059       emit_int8(0x66);
 4060     }
 4061     i -= 3;
 4062     emit_int24(0x66, 0x66, (unsigned char)0x90);
 4063   }
 4064   switch (i) {
 4065     case 4:
 4066       emit_int8(0x66);
 4067     case 3:
 4068       emit_int8(0x66);
 4069     case 2:
 4070       emit_int8(0x66);
 4071     case 1:
 4072       emit_int8((unsigned char)0x90);
 4073       break;
 4074     default:
 4075       assert(i == 0, " ");
 4076   }
 4077 }
 4078 
 4079 void Assembler::notl(Register dst) {
 4080   int encode = prefix_and_encode(dst->encoding());
 4081   emit_int16((unsigned char)0xF7, (0xD0 | encode));
 4082 }
 4083 
 4084 void Assembler::orw(Register dst, Register src) {
 4085   (void)prefix_and_encode(dst->encoding(), src->encoding());
 4086   emit_arith(0x0B, 0xC0, dst, src);
 4087 }
 4088 
 4089 void Assembler::orl(Address dst, int32_t imm32) {
 4090   InstructionMark im(this);
 4091   prefix(dst);
 4092   emit_arith_operand(0x81, rcx, dst, imm32);
 4093 }
 4094 
 4095 void Assembler::orl(Register dst, int32_t imm32) {
 4096   prefix(dst);
 4097   emit_arith(0x81, 0xC8, dst, imm32);
 4098 }
 4099 
 4100 void Assembler::orl(Register dst, Address src) {
 4101   InstructionMark im(this);
 4102   prefix(src, dst);
 4103   emit_int8(0x0B);
 4104   emit_operand(dst, src, 0);
 4105 }
 4106 
 4107 void Assembler::orl(Register dst, Register src) {
 4108   (void) prefix_and_encode(dst->encoding(), src->encoding());
 4109   emit_arith(0x0B, 0xC0, dst, src);
 4110 }
 4111 
 4112 void Assembler::orl(Address dst, Register src) {
 4113   InstructionMark im(this);
 4114   prefix(dst, src);
 4115   emit_int8(0x09);
 4116   emit_operand(src, dst, 0);
 4117 }
 4118 
 4119 void Assembler::orb(Address dst, int imm8) {
 4120   InstructionMark im(this);
 4121   prefix(dst);
 4122   emit_int8((unsigned char)0x80);
 4123   emit_operand(rcx, dst, 1);
 4124   emit_int8(imm8);
 4125 }
 4126 
 4127 void Assembler::orb(Address dst, Register src) {
 4128   InstructionMark im(this);
 4129   prefix(dst, src, true);
 4130   emit_int8(0x08);
 4131   emit_operand(src, dst, 0);
 4132 }
 4133 
 4134 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
 4135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4137   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4138   emit_int16(0x63, (0xC0 | encode));
 4139 }
 4140 
 4141 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4142   assert(UseAVX > 0, "some form of AVX must be enabled");
 4143   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4144   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4145   emit_int16(0x63, (0xC0 | encode));
 4146 }
 4147 
 4148 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
 4149   assert(VM_Version::supports_sse2(), "");
 4150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4151   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4152   emit_int16(0x6B, (0xC0 | encode));
 4153 }
 4154 
 4155 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4156   assert(UseAVX > 0, "some form of AVX must be enabled");
 4157   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4158   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4159   emit_int16(0x6B, (0xC0 | encode));
 4160 }
 4161 
 4162 void Assembler::packuswb(XMMRegister dst, Address src) {
 4163   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4164   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 4165   InstructionMark im(this);
 4166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4167   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4168   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4169   emit_int8(0x67);
 4170   emit_operand(dst, src, 0);
 4171 }
 4172 
 4173 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 4174   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 4175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4176   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4177   emit_int16(0x67, (0xC0 | encode));
 4178 }
 4179 
 4180 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4181   assert(UseAVX > 0, "some form of AVX must be enabled");
 4182   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4183   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4184   emit_int16(0x67, (0xC0 | encode));
 4185 }
 4186 
 4187 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
 4188   assert(VM_Version::supports_sse4_1(), "");
 4189   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4190   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4191   emit_int16(0x2B, (0xC0 | encode));
 4192 }
 4193 
 4194 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4195   assert(UseAVX > 0, "some form of AVX must be enabled");
 4196   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4197   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4198   emit_int16(0x2B, (0xC0 | encode));
 4199 }
 4200 
 4201 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4202   assert(VM_Version::supports_avx2(), "");
 4203   assert(vector_len != AVX_128bit, "");
 4204   // VEX.256.66.0F3A.W1 00 /r ib
 4205   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4206   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4207   emit_int24(0x00, (0xC0 | encode), imm8);
 4208 }
 4209 
 4210 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4211   assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
 4212          vector_len == AVX_512bit ? VM_Version::supports_evex()     : false, "not supported");
 4213   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4214   attributes.set_is_evex_instruction();
 4215   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4216   emit_int16(0x36, (0xC0 | encode));
 4217 }
 4218 
 4219 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4220   assert(VM_Version::supports_avx512_vbmi(), "");
 4221   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4222   attributes.set_is_evex_instruction();
 4223   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4224   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4225 }
 4226 
 4227 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4228   assert(VM_Version::supports_avx512_vbmi(), "");
 4229   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4230   attributes.set_is_evex_instruction();
 4231   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4232   emit_int8((unsigned char)0x8D);
 4233   emit_operand(dst, src, 0);
 4234 }
 4235 
 4236 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4237   assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
 4238          vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
 4239          vector_len == AVX_512bit ? VM_Version::supports_avx512bw()   : false, "not supported");
 4240   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4241   attributes.set_is_evex_instruction();
 4242   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4243   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 4244 }
 4245 
 4246 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4247   assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 4248          (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
 4249   // VEX.NDS.256.66.0F38.W0 36 /r
 4250   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4251   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4252   emit_int16(0x36, (0xC0 | encode));
 4253 }
 4254 
 4255 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 4256   assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 4257          (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
 4258   // VEX.NDS.256.66.0F38.W0 36 /r
 4259   InstructionMark im(this);
 4260   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4261   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4262   emit_int8(0x36);
 4263   emit_operand(dst, src, 0);
 4264 }
 4265 
 4266 void Assembler::vpermps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4267   assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 4268          (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
 4269   // VEX.NDS.XXX.66.0F38.W0 16 /r
 4270   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4271   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4272   emit_int16(0x16, (0xC0 | encode));
 4273 }
 4274 
 4275 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
 4276   assert(VM_Version::supports_avx2(), "");
 4277   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4278   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4279   emit_int24(0x46, (0xC0 | encode), imm8);
 4280 }
 4281 
 4282 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4283   assert(VM_Version::supports_avx(), "");
 4284   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4285   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4286   emit_int24(0x06, (0xC0 | encode), imm8);
 4287 }
 4288 
 4289 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4290   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4291   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4292   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4293   emit_int24(0x04, (0xC0 | encode), imm8);
 4294 }
 4295 
 4296 void Assembler::vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4297   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4298   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4299   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4300   emit_int16(0x0C, (0xC0 | encode));
 4301 }
 4302 
 4303 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4304   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
 4305   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
 4306   attributes.set_rex_vex_w_reverted();
 4307   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4308   emit_int24(0x05, (0xC0 | encode), imm8);
 4309 }
 4310 
 4311 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
 4312   assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
 4313   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
 4314   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4315   emit_int24(0x01, (0xC0 | encode), imm8);
 4316 }
 4317 
 4318 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4319   assert(VM_Version::supports_evex(), "");
 4320   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4321   attributes.set_is_evex_instruction();
 4322   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4323   emit_int16(0x76, (0xC0 | encode));
 4324 }
 4325 
 4326 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4327   assert(VM_Version::supports_avx512_vbmi(), "");
 4328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4329   attributes.set_is_evex_instruction();
 4330   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4331   emit_int16(0x7D, (0xC0 | encode));
 4332 }
 4333 
 4334 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
 4335   assert(VM_Version::supports_avx512_vbmi(), "");
 4336   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4337   attributes.set_is_evex_instruction();
 4338   int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4339   emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
 4340 }
 4341 
 4342 void Assembler::pause() {
 4343   emit_int16((unsigned char)0xF3, (unsigned char)0x90);
 4344 }
 4345 
 4346 void Assembler::ud2() {
 4347   emit_int16(0x0F, 0x0B);
 4348 }
 4349 
 4350 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 4351   assert(VM_Version::supports_sse4_2(), "");
 4352   InstructionMark im(this);
 4353   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4354   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4355   emit_int8(0x61);
 4356   emit_operand(dst, src, 1);
 4357   emit_int8(imm8);
 4358 }
 4359 
 4360 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 4361   assert(VM_Version::supports_sse4_2(), "");
 4362   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4363   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4364   emit_int24(0x61, (0xC0 | encode), imm8);
 4365 }
 4366 
 4367 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4368 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 4369   assert(VM_Version::supports_sse2(), "");
 4370   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4371   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4372   emit_int16(0x74, (0xC0 | encode));
 4373 }
 4374 
 4375 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4376   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4377   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4378   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4379   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4380   emit_int16(cond_encoding, (0xC0 | encode));
 4381 }
 4382 
 4383 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4384 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4385   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4386   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4387   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4388   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4389   emit_int16(0x74, (0xC0 | encode));
 4390 }
 4391 
 4392 // In this context, kdst is written the mask used to process the equal components
 4393 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4394   assert(VM_Version::supports_avx512bw(), "");
 4395   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4396   attributes.set_is_evex_instruction();
 4397   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4398   emit_int16(0x74, (0xC0 | encode));
 4399 }
 4400 
 4401 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4402   assert(VM_Version::supports_avx512vlbw(), "");
 4403   InstructionMark im(this);
 4404   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4405   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4406   attributes.set_is_evex_instruction();
 4407   int dst_enc = kdst->encoding();
 4408   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4409   emit_int8(0x64);
 4410   emit_operand(as_Register(dst_enc), src, 0);
 4411 }
 4412 
 4413 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4414   assert(VM_Version::supports_avx512vlbw(), "");
 4415   InstructionMark im(this);
 4416   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4417   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4418   attributes.reset_is_clear_context();
 4419   attributes.set_embedded_opmask_register_specifier(mask);
 4420   attributes.set_is_evex_instruction();
 4421   int dst_enc = kdst->encoding();
 4422   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4423   emit_int8(0x64);
 4424   emit_operand(as_Register(dst_enc), src, 0);
 4425 }
 4426 
 4427 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4428   assert(VM_Version::supports_avx512vlbw(), "");
 4429   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4430   attributes.set_is_evex_instruction();
 4431   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4432   emit_int24(0x3E, (0xC0 | encode), vcc);
 4433 }
 4434 
 4435 void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
 4436   assert(VM_Version::supports_avx512vl(), "");
 4437   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4438   attributes.set_is_evex_instruction();
 4439   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4440   emit_int24(0x1E, (0xC0 | encode), vcc);
 4441 }
 4442 
 4443 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
 4444   assert(VM_Version::supports_avx512vlbw(), "");
 4445   InstructionMark im(this);
 4446   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4447   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4448   attributes.set_is_evex_instruction();
 4449   int dst_enc = kdst->encoding();
 4450   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4451   emit_int8(0x3E);
 4452   emit_operand(as_Register(dst_enc), src, 1);
 4453   emit_int8(vcc);
 4454 }
 4455 
 4456 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4457   assert(VM_Version::supports_avx512bw(), "");
 4458   InstructionMark im(this);
 4459   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4460   attributes.set_is_evex_instruction();
 4461   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4462   int dst_enc = kdst->encoding();
 4463   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4464   emit_int8(0x74);
 4465   emit_operand(as_Register(dst_enc), src, 0);
 4466 }
 4467 
 4468 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4469   assert(VM_Version::supports_avx512vlbw(), "");
 4470   InstructionMark im(this);
 4471   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4472   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4473   attributes.reset_is_clear_context();
 4474   attributes.set_embedded_opmask_register_specifier(mask);
 4475   attributes.set_is_evex_instruction();
 4476   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4477   emit_int8(0x74);
 4478   emit_operand(as_Register(kdst->encoding()), src, 0);
 4479 }
 4480 
 4481 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4482 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 4483   assert(VM_Version::supports_sse2(), "");
 4484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4485   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4486   emit_int16(0x75, (0xC0 | encode));
 4487 }
 4488 
 4489 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4490 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4491   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4492   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4493   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4494   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4495   emit_int16(0x75, (0xC0 | encode));
 4496 }
 4497 
 4498 // In this context, kdst is written the mask used to process the equal components
 4499 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4500   assert(VM_Version::supports_avx512bw(), "");
 4501   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4502   attributes.set_is_evex_instruction();
 4503   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4504   emit_int16(0x75, (0xC0 | encode));
 4505 }
 4506 
 4507 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4508   assert(VM_Version::supports_avx512bw(), "");
 4509   InstructionMark im(this);
 4510   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4511   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 4512   attributes.set_is_evex_instruction();
 4513   int dst_enc = kdst->encoding();
 4514   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4515   emit_int8(0x75);
 4516   emit_operand(as_Register(dst_enc), src, 0);
 4517 }
 4518 
 4519 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4520 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
 4521   assert(VM_Version::supports_sse2(), "");
 4522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4523   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4524   emit_int16(0x76, (0xC0 | encode));
 4525 }
 4526 
 4527 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4528 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4529   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
 4530   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
 4531   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4532   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4533   emit_int16(0x76, (0xC0 | encode));
 4534 }
 4535 
 4536 // In this context, kdst is written the mask used to process the equal components
 4537 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4538   assert(VM_Version::supports_evex(), "");
 4539   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4540   attributes.set_is_evex_instruction();
 4541   attributes.reset_is_clear_context();
 4542   attributes.set_embedded_opmask_register_specifier(mask);
 4543   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4544   emit_int16(0x76, (0xC0 | encode));
 4545 }
 4546 
 4547 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
 4548   assert(VM_Version::supports_evex(), "");
 4549   InstructionMark im(this);
 4550   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4551   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 4552   attributes.set_is_evex_instruction();
 4553   attributes.reset_is_clear_context();
 4554   attributes.set_embedded_opmask_register_specifier(mask);
 4555   int dst_enc = kdst->encoding();
 4556   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4557   emit_int8(0x76);
 4558   emit_operand(as_Register(dst_enc), src, 0);
 4559 }
 4560 
 4561 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4562 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
 4563   assert(VM_Version::supports_sse4_1(), "");
 4564   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4565   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4566   emit_int16(0x29, (0xC0 | encode));
 4567 }
 4568 
 4569 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
 4570   assert(VM_Version::supports_evex(), "");
 4571   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4572   attributes.set_is_evex_instruction();
 4573   attributes.reset_is_clear_context();
 4574   attributes.set_embedded_opmask_register_specifier(mask);
 4575   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4576   emit_int16(0x29, (0xC0 | encode));
 4577 }
 4578 
 4579 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
 4580   assert(VM_Version::supports_avx(), "");
 4581   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4582   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4583   emit_int16(cond_encoding, (0xC0 | encode));
 4584 }
 4585 
 4586 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
 4587 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 4588   assert(VM_Version::supports_avx(), "");
 4589   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4590   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4591   emit_int16(0x29, (0xC0 | encode));
 4592 }
 4593 
 4594 // In this context, kdst is written the mask used to process the equal components
 4595 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
 4596   assert(VM_Version::supports_evex(), "");
 4597   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4598   attributes.reset_is_clear_context();
 4599   attributes.set_is_evex_instruction();
 4600   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4601   emit_int16(0x29, (0xC0 | encode));
 4602 }
 4603 
 4604 // In this context, kdst is written the mask used to process the equal components
 4605 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
 4606   assert(VM_Version::supports_evex(), "");
 4607   InstructionMark im(this);
 4608   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4609   attributes.reset_is_clear_context();
 4610   attributes.set_is_evex_instruction();
 4611   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 4612   int dst_enc = kdst->encoding();
 4613   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4614   emit_int8(0x29);
 4615   emit_operand(as_Register(dst_enc), src, 0);
 4616 }
 4617 
 4618 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
 4619   assert(VM_Version::supports_sse4_1(), "");
 4620   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4621   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4622   emit_int16(0x37, (0xC0 | encode));
 4623 }
 4624 
 4625 void Assembler::pmovmskb(Register dst, XMMRegister src) {
 4626   assert(VM_Version::supports_sse2(), "");
 4627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4628   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4629   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4630 }
 4631 
 4632 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
 4633   assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
 4634          (VM_Version::supports_avx2() && vec_enc  == AVX_256bit), "");
 4635   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4636   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4637   emit_int16((unsigned char)0xD7, (0xC0 | encode));
 4638 }
 4639 
 4640 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
 4641   assert(VM_Version::supports_avx(), "");
 4642   InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4643   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 4644   emit_int16(0x50, (0xC0 | encode));
 4645 }
 4646 
 4647 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
 4648   assert(VM_Version::supports_avx(), "");
 4649   InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 4650   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4651   emit_int16(0x50, (0xC0 | encode));
 4652 }
 4653 
 4654 
 4655 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
 4656   assert(VM_Version::supports_sse4_1(), "");
 4657   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4658   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4659   emit_int24(0x16, (0xC0 | encode), imm8);
 4660 }
 4661 
 4662 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
 4663   assert(VM_Version::supports_sse4_1(), "");
 4664   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4665   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4666   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4667   emit_int8(0x16);
 4668   emit_operand(src, dst, 1);
 4669   emit_int8(imm8);
 4670 }
 4671 
 4672 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
 4673   assert(VM_Version::supports_sse4_1(), "");
 4674   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4675   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4676   emit_int24(0x16, (0xC0 | encode), imm8);
 4677 }
 4678 
 4679 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
 4680   assert(VM_Version::supports_sse4_1(), "");
 4681   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4682   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4683   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4684   emit_int8(0x16);
 4685   emit_operand(src, dst, 1);
 4686   emit_int8(imm8);
 4687 }
 4688 
 4689 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
 4690   assert(VM_Version::supports_sse2(), "");
 4691   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4692   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4693   emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
 4694 }
 4695 
 4696 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
 4697   assert(VM_Version::supports_sse4_1(), "");
 4698   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4699   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4700   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4701   emit_int8(0x15);
 4702   emit_operand(src, dst, 1);
 4703   emit_int8(imm8);
 4704 }
 4705 
 4706 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
 4707   assert(VM_Version::supports_sse4_1(), "");
 4708   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4709   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4710   emit_int24(0x14, (0xC0 | encode), imm8);
 4711 }
 4712 
 4713 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
 4714   assert(VM_Version::supports_sse4_1(), "");
 4715   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4716   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4717   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4718   emit_int8(0x14);
 4719   emit_operand(src, dst, 1);
 4720   emit_int8(imm8);
 4721 }
 4722 
 4723 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
 4724   assert(VM_Version::supports_sse4_1(), "");
 4725   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4726   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4727   emit_int24(0x22, (0xC0 | encode), imm8);
 4728 }
 4729 
 4730 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
 4731   assert(VM_Version::supports_sse4_1(), "");
 4732   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4733   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 4734   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4735   emit_int8(0x22);
 4736   emit_operand(dst, src, 1);
 4737   emit_int8(imm8);
 4738 }
 4739 
 4740 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4741   assert(VM_Version::supports_avx(), "");
 4742   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4743   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4744   emit_int24(0x22, (0xC0 | encode), imm8);
 4745 }
 4746 
 4747 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
 4748   assert(VM_Version::supports_sse4_1(), "");
 4749   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4750   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4751   emit_int24(0x22, (0xC0 | encode), imm8);
 4752 }
 4753 
 4754 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
 4755   assert(VM_Version::supports_sse4_1(), "");
 4756   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4757   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 4758   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4759   emit_int8(0x22);
 4760   emit_operand(dst, src, 1);
 4761   emit_int8(imm8);
 4762 }
 4763 
 4764 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4765   assert(VM_Version::supports_avx(), "");
 4766   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
 4767   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4768   emit_int24(0x22, (0xC0 | encode), imm8);
 4769 }
 4770 
 4771 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
 4772   assert(VM_Version::supports_sse2(), "");
 4773   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4774   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4775   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4776 }
 4777 
 4778 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
 4779   assert(VM_Version::supports_sse2(), "");
 4780   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 4782   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4783   emit_int8((unsigned char)0xC4);
 4784   emit_operand(dst, src, 1);
 4785   emit_int8(imm8);
 4786 }
 4787 
 4788 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4789   assert(VM_Version::supports_avx(), "");
 4790   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4792   emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
 4793 }
 4794 
 4795 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
 4796   assert(VM_Version::supports_sse4_1(), "");
 4797   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4798   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 4799   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4800   emit_int8(0x20);
 4801   emit_operand(dst, src, 1);
 4802   emit_int8(imm8);
 4803 }
 4804 
 4805 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
 4806   assert(VM_Version::supports_sse4_1(), "");
 4807   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4808   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4809   emit_int24(0x20, (0xC0 | encode), imm8);
 4810 }
 4811 
 4812 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
 4813   assert(VM_Version::supports_avx(), "");
 4814   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
 4815   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4816   emit_int24(0x20, (0xC0 | encode), imm8);
 4817 }
 4818 
 4819 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
 4820   assert(VM_Version::supports_sse4_1(), "");
 4821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4822   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4823   emit_int24(0x21, (0xC0 | encode), imm8);
 4824 }
 4825 
 4826 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
 4827   assert(VM_Version::supports_avx(), "");
 4828   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 4829   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 4830   emit_int24(0x21, (0xC0 | encode), imm8);
 4831 }
 4832 
 4833 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
 4834   assert(VM_Version::supports_sse4_1(), "");
 4835   InstructionMark im(this);
 4836   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4837   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4838   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4839   emit_int8(0x30);
 4840   emit_operand(dst, src, 0);
 4841 }
 4842 
 4843 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 4844   assert(VM_Version::supports_sse4_1(), "");
 4845   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4846   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4847   emit_int16(0x30, (0xC0 | encode));
 4848 }
 4849 
 4850 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
 4851   assert(VM_Version::supports_sse4_1(), "");
 4852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4853   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4854   emit_int16(0x20, (0xC0 | encode));
 4855 }
 4856 
 4857 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
 4858   assert(VM_Version::supports_sse4_1(), "");
 4859   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4860   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4861   emit_int16(0x35, (0xC0 | encode));
 4862 }
 4863 
 4864 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
 4865   assert(VM_Version::supports_sse4_1(), "");
 4866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4867   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4868   emit_int16(0x21, (0xC0 | encode));
 4869 }
 4870 
 4871 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
 4872   assert(VM_Version::supports_sse4_1(), "");
 4873   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4874   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4875   emit_int16(0x31, (0xC0 | encode));
 4876 }
 4877 
 4878 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
 4879   assert(VM_Version::supports_sse4_1(), "");
 4880   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4881   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4882   emit_int16(0x22, (0xC0 | encode));
 4883 }
 4884 
 4885 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
 4886   assert(VM_Version::supports_sse4_1(), "");
 4887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4889   emit_int16(0x23, (0xC0 | encode));
 4890 }
 4891 
 4892 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 4893   assert(VM_Version::supports_avx(), "");
 4894   InstructionMark im(this);
 4895   assert(dst != xnoreg, "sanity");
 4896   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4897   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4898   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4899   emit_int8(0x30);
 4900   emit_operand(dst, src, 0);
 4901 }
 4902 
 4903 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4904   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4905   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4906   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4907   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4908   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4909   emit_int16(0x30, (unsigned char) (0xC0 | encode));
 4910 }
 4911 
 4912 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
 4913   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 4914   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 4915   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 4916   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 4917   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4918   emit_int16(0x20, (0xC0 | encode));
 4919 }
 4920 
 4921 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4922   assert(VM_Version::supports_avx512vlbw(), "");
 4923   assert(dst != xnoreg, "sanity");
 4924   InstructionMark im(this);
 4925   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4926   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4927   attributes.set_embedded_opmask_register_specifier(mask);
 4928   attributes.set_is_evex_instruction();
 4929   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4930   emit_int8(0x30);
 4931   emit_operand(dst, src, 0);
 4932 }
 4933 
 4934 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
 4935   assert(VM_Version::supports_avx512vl(), "");
 4936   assert(dst != xnoreg, "sanity");
 4937   InstructionMark im(this);
 4938   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
 4939   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 4940   attributes.set_embedded_opmask_register_specifier(mask);
 4941   attributes.set_is_evex_instruction();
 4942   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4943   emit_int8(0x31);
 4944   emit_operand(dst, src, 0);
 4945 }
 4946 
 4947 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) {
 4948   evpmovzxbd(dst, k0, src, vector_len);
 4949 }
 4950 
 4951 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 4952   assert(VM_Version::supports_evex(), "");
 4953   // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
 4954   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 4955   attributes.set_is_evex_instruction();
 4956   attributes.set_embedded_opmask_register_specifier(mask);
 4957   if (merge) {
 4958     attributes.reset_is_clear_context();
 4959   }
 4960   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 4961   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 4962 }
 4963 
 4964 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 4965   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4966   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4967   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4968   emit_int16(0x35, (0xC0 | encode));
 4969 }
 4970 
 4971 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4972   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4973   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4974   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4975   emit_int16(0x31, (0xC0 | encode));
 4976 }
 4977 
 4978 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4979   assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
 4980   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4981   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4982   emit_int16(0x32, (0xC0 | encode));
 4983 }
 4984 
 4985 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
 4986   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4987          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4988              VM_Version::supports_evex(), "");
 4989   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4990   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 4991   emit_int16(0x21, (0xC0 | encode));
 4992 }
 4993 
 4994 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
 4995   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 4996          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 4997              VM_Version::supports_evex(), "");
 4998   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 4999   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5000   emit_int16(0x22, (0xC0 | encode));
 5001 }
 5002 
 5003 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 5004   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5005          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5006              VM_Version::supports_evex(), "");
 5007   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5008   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5009   emit_int16(0x23, (0xC0 | encode));
 5010 }
 5011 
 5012 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 5013   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5014          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5015              VM_Version::supports_evex(), "");
 5016   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5017   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5018   emit_int16(0x24, (0xC0 | encode));
 5019 }
 5020 
 5021 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
 5022   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5023          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5024              VM_Version::supports_evex(), "");
 5025   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5026   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5027   emit_int16(0x25, (0xC0 | encode));
 5028 }
 5029 
 5030 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
 5031   assert(VM_Version::supports_avx512vlbw(), "");
 5032   assert(src != xnoreg, "sanity");
 5033   InstructionMark im(this);
 5034   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5035   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 5036   attributes.set_is_evex_instruction();
 5037   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5038   emit_int8(0x30);
 5039   emit_operand(src, dst, 0);
 5040 }
 5041 
 5042 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
 5043   assert(VM_Version::supports_avx512vlbw(), "");
 5044   assert(src != xnoreg, "sanity");
 5045   InstructionMark im(this);
 5046   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5047   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
 5048   attributes.reset_is_clear_context();
 5049   attributes.set_embedded_opmask_register_specifier(mask);
 5050   attributes.set_is_evex_instruction();
 5051   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5052   emit_int8(0x30);
 5053   emit_operand(src, dst, 0);
 5054 }
 5055 
 5056 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
 5057   assert(VM_Version::supports_evex(), "");
 5058   assert(src != xnoreg, "sanity");
 5059   InstructionMark im(this);
 5060   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5061   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
 5062   attributes.set_is_evex_instruction();
 5063   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5064   emit_int8(0x31);
 5065   emit_operand(src, dst, 0);
 5066 }
 5067 
 5068 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
 5069   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5070   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5071   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 5072   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5073   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5074   emit_int16(0x33, (0xC0 | encode));
 5075 }
 5076 
 5077 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
 5078   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5079   vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5080   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
 5081   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5082   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5083   emit_int16(0x34, (0xC0 | encode));
 5084 }
 5085 
 5086 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
 5087   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5089   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5090   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5091 }
 5092 
 5093 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5094   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5095     (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5096     (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
 5097   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5098   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5099   emit_int16((unsigned char)0xF5, (0xC0 | encode));
 5100 }
 5101 
 5102 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5103 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5104        vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5105        vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5106   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5107   int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5108   emit_int16(0x04, (0xC0 | encode));
 5109 }
 5110 
 5111 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5112   evpmadd52luq(dst, k0, src1, src2, false, vector_len);
 5113 }
 5114 
 5115 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5116   assert(VM_Version::supports_avx512ifma(), "");
 5117   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5118   attributes.set_is_evex_instruction();
 5119   attributes.set_embedded_opmask_register_specifier(mask);
 5120   if (merge) {
 5121     attributes.reset_is_clear_context();
 5122   }
 5123 
 5124   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5125   emit_int16((unsigned char)0xB4, (0xC0 | encode));
 5126 }
 5127 
 5128 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5129   evpmadd52huq(dst, k0, src1, src2, false, vector_len);
 5130 }
 5131 
 5132 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5133   assert(VM_Version::supports_avx512ifma(), "");
 5134   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5135   attributes.set_is_evex_instruction();
 5136   attributes.set_embedded_opmask_register_specifier(mask);
 5137   if (merge) {
 5138     attributes.reset_is_clear_context();
 5139   }
 5140 
 5141   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5142   emit_int16((unsigned char)0xB5, (0xC0 | encode));
 5143 }
 5144 
 5145 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5146   assert(VM_Version::supports_evex(), "");
 5147   assert(VM_Version::supports_avx512_vnni(), "must support vnni");
 5148   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5149   attributes.set_is_evex_instruction();
 5150   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5151   emit_int16(0x52, (0xC0 | encode));
 5152 }
 5153 
 5154 // generic
 5155 void Assembler::pop(Register dst) {
 5156   int encode = prefix_and_encode(dst->encoding());
 5157   emit_int8(0x58 | encode);
 5158 }
 5159 
 5160 void Assembler::popcntl(Register dst, Address src) {
 5161   assert(VM_Version::supports_popcnt(), "must support");
 5162   InstructionMark im(this);
 5163   emit_int8((unsigned char)0xF3);
 5164   prefix(src, dst);
 5165   emit_int16(0x0F, (unsigned char)0xB8);
 5166   emit_operand(dst, src, 0);
 5167 }
 5168 
 5169 void Assembler::popcntl(Register dst, Register src) {
 5170   assert(VM_Version::supports_popcnt(), "must support");
 5171   emit_int8((unsigned char)0xF3);
 5172   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 5173   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
 5174 }
 5175 
 5176 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5177   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5178   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5179   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5180   attributes.set_embedded_opmask_register_specifier(mask);
 5181   attributes.set_is_evex_instruction();
 5182   if (merge) {
 5183     attributes.reset_is_clear_context();
 5184   }
 5185   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5186   emit_int16(0x54, (0xC0 | encode));
 5187 }
 5188 
 5189 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5190   assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
 5191   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5192   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5193   attributes.set_is_evex_instruction();
 5194   attributes.set_embedded_opmask_register_specifier(mask);
 5195   if (merge) {
 5196     attributes.reset_is_clear_context();
 5197   }
 5198   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5199   emit_int16(0x54, (0xC0 | encode));
 5200 }
 5201 
 5202 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5203   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5204   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5206   attributes.set_is_evex_instruction();
 5207   attributes.set_embedded_opmask_register_specifier(mask);
 5208   if (merge) {
 5209     attributes.reset_is_clear_context();
 5210   }
 5211   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5212   emit_int16(0x55, (0xC0 | encode));
 5213 }
 5214 
 5215 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 5216   assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
 5217   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 5218   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5219   attributes.set_is_evex_instruction();
 5220   attributes.set_embedded_opmask_register_specifier(mask);
 5221   if (merge) {
 5222     attributes.reset_is_clear_context();
 5223   }
 5224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5225   emit_int16(0x55, (0xC0 | encode));
 5226 }
 5227 
 5228 void Assembler::popf() {
 5229   emit_int8((unsigned char)0x9D);
 5230 }
 5231 
 5232 #ifndef _LP64 // no 32bit push/pop on amd64
 5233 void Assembler::popl(Address dst) {
 5234   // NOTE: this will adjust stack by 8byte on 64bits
 5235   InstructionMark im(this);
 5236   prefix(dst);
 5237   emit_int8((unsigned char)0x8F);
 5238   emit_operand(rax, dst, 0);
 5239 }
 5240 #endif
 5241 
 5242 void Assembler::prefetchnta(Address src) {
 5243   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5244   InstructionMark im(this);
 5245   prefix(src);
 5246   emit_int16(0x0F, 0x18);
 5247   emit_operand(rax, src, 0); // 0, src
 5248 }
 5249 
 5250 void Assembler::prefetchr(Address src) {
 5251   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5252   InstructionMark im(this);
 5253   prefix(src);
 5254   emit_int16(0x0F, 0x0D);
 5255   emit_operand(rax, src, 0); // 0, src
 5256 }
 5257 
 5258 void Assembler::prefetcht0(Address src) {
 5259   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5260   InstructionMark im(this);
 5261   prefix(src);
 5262   emit_int16(0x0F, 0x18);
 5263   emit_operand(rcx, src, 0); // 1, src
 5264 }
 5265 
 5266 void Assembler::prefetcht1(Address src) {
 5267   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5268   InstructionMark im(this);
 5269   prefix(src);
 5270   emit_int16(0x0F, 0x18);
 5271   emit_operand(rdx, src, 0); // 2, src
 5272 }
 5273 
 5274 void Assembler::prefetcht2(Address src) {
 5275   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
 5276   InstructionMark im(this);
 5277   prefix(src);
 5278   emit_int16(0x0F, 0x18);
 5279   emit_operand(rbx, src, 0); // 3, src
 5280 }
 5281 
 5282 void Assembler::prefetchw(Address src) {
 5283   assert(VM_Version::supports_3dnow_prefetch(), "must support");
 5284   InstructionMark im(this);
 5285   prefix(src);
 5286   emit_int16(0x0F, 0x0D);
 5287   emit_operand(rcx, src, 0); // 1, src
 5288 }
 5289 
 5290 void Assembler::prefix(Prefix p) {
 5291   emit_int8(p);
 5292 }
 5293 
 5294 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
 5295   assert(VM_Version::supports_ssse3(), "");
 5296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5298   emit_int16(0x00, (0xC0 | encode));
 5299 }
 5300 
 5301 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 5302   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 5303   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5304   attributes.set_is_evex_instruction();
 5305   attributes.set_embedded_opmask_register_specifier(mask);
 5306   if (merge) {
 5307     attributes.reset_is_clear_context();
 5308   }
 5309   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5310   emit_int16(0x00, (0xC0 | encode));
 5311 }
 5312 
 5313 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5314   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5315          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5316          vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
 5317   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5318   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5319   emit_int16(0x00, (0xC0 | encode));
 5320 }
 5321 
 5322 void Assembler::pshufb(XMMRegister dst, Address src) {
 5323   assert(VM_Version::supports_ssse3(), "");
 5324   InstructionMark im(this);
 5325   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5326   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5327   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5328   emit_int8(0x00);
 5329   emit_operand(dst, src, 0);
 5330 }
 5331 
 5332 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 5333   assert(isByte(mode), "invalid value");
 5334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5335   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
 5336   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5337   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5338   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5339 }
 5340 
 5341 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5342   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5343          (vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5344          (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
 5345   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5346   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5347   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5348   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5349 }
 5350 
 5351 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 5352   assert(isByte(mode), "invalid value");
 5353   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5354   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5355   InstructionMark im(this);
 5356   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5357   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5358   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5359   emit_int8(0x70);
 5360   emit_operand(dst, src, 1);
 5361   emit_int8(mode & 0xFF);
 5362 }
 5363 
 5364 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
 5365   assert(isByte(mode), "invalid value");
 5366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5368   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5369   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5370 }
 5371 
 5372 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5373     assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5374             (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5375             (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
 5376     NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5377     InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5378     int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5379     emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5380 }
 5381 
 5382 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 5383   assert(isByte(mode), "invalid value");
 5384   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5386   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5387   emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5388 }
 5389 
 5390 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 5391   assert(isByte(mode), "invalid value");
 5392   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5393   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5394   InstructionMark im(this);
 5395   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5396   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5397   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5398   emit_int8(0x70);
 5399   emit_operand(dst, src, 1);
 5400   emit_int8(mode & 0xFF);
 5401 }
 5402 
 5403 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
 5404     assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5405             (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5406             (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
 5407     NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5408     InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5409     int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 5410     emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
 5411 }
 5412 
 5413 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5414   assert(VM_Version::supports_evex(), "requires EVEX support");
 5415   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
 5416   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5417   attributes.set_is_evex_instruction();
 5418   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5419   emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
 5420 }
 5421 
 5422 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
 5423   assert(isByte(imm8), "invalid value");
 5424   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5425   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5426   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5427   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5428 }
 5429 
 5430 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5431   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5432   attributes.set_rex_vex_w_reverted();
 5433   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5434   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5435 }
 5436 
 5437 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
 5438   assert(isByte(imm8), "invalid value");
 5439   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5440   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5441   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5442   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5443 }
 5444 
 5445 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5446   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5447   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5448   emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
 5449 }
 5450 
 5451 void Assembler::psrldq(XMMRegister dst, int shift) {
 5452   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5453   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5454   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5455   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5456   emit_int24(0x73, (0xC0 | encode), shift);
 5457 }
 5458 
 5459 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5460   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5461          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5462          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5463   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5464   int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5465   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5466 }
 5467 
 5468 void Assembler::pslldq(XMMRegister dst, int shift) {
 5469   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
 5470   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5471   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5472   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
 5473   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5474   emit_int24(0x73, (0xC0 | encode), shift);
 5475 }
 5476 
 5477 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 5478   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 5479          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 5480          vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
 5481   InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5482   int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5483   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 5484 }
 5485 
 5486 void Assembler::ptest(XMMRegister dst, Address src) {
 5487   assert(VM_Version::supports_sse4_1(), "");
 5488   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5489   InstructionMark im(this);
 5490   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5491   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5492   emit_int8(0x17);
 5493   emit_operand(dst, src, 0);
 5494 }
 5495 
 5496 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 5497   assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
 5498   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5499   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5500   emit_int8(0x17);
 5501   emit_int8((0xC0 | encode));
 5502 }
 5503 
 5504 void Assembler::vptest(XMMRegister dst, Address src) {
 5505   assert(VM_Version::supports_avx(), "");
 5506   InstructionMark im(this);
 5507   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5508   assert(dst != xnoreg, "sanity");
 5509   // swap src<->dst for encoding
 5510   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5511   emit_int8(0x17);
 5512   emit_operand(dst, src, 0);
 5513 }
 5514 
 5515 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
 5516   assert(VM_Version::supports_avx(), "");
 5517   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5518   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5519   emit_int16(0x17, (0xC0 | encode));
 5520 }
 5521 
 5522 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
 5523   assert(VM_Version::supports_avx(), "");
 5524   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5525   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5526   emit_int16(0x17, (0xC0 | encode));
 5527 }
 5528 
 5529 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) {
 5530   assert(VM_Version::supports_avx(), "");
 5531   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5532   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5533   emit_int16(0x0E, (0xC0 | encode));
 5534 }
 5535 
 5536 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5537   assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), "");
 5538   // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
 5539   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5540   attributes.set_is_evex_instruction();
 5541   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5542   emit_int16(0x26, (0xC0 | encode));
 5543 }
 5544 
 5545 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5546   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
 5547   // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
 5548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5549   attributes.set_is_evex_instruction();
 5550   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 5551   emit_int16(0x27, (0xC0 | encode));
 5552 }
 5553 
 5554 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 5555   assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
 5556   // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r
 5557   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5558   attributes.set_is_evex_instruction();
 5559   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
 5560   emit_int16(0x27, (0xC0 | encode));
 5561 }
 5562 
 5563 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 5564   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5565   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5566   InstructionMark im(this);
 5567   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5568   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 5569   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5570   emit_int8(0x60);
 5571   emit_operand(dst, src, 0);
 5572 }
 5573 
 5574 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 5575   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5576   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
 5577   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5578   emit_int16(0x60, (0xC0 | encode));
 5579 }
 5580 
 5581 void Assembler::punpckldq(XMMRegister dst, Address src) {
 5582   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5583   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 5584   InstructionMark im(this);
 5585   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5586   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 5587   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5588   emit_int8(0x62);
 5589   emit_operand(dst, src, 0);
 5590 }
 5591 
 5592 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 5593   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5594   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5595   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5596   emit_int16(0x62, (0xC0 | encode));
 5597 }
 5598 
 5599 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 5600   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 5601   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5602   attributes.set_rex_vex_w_reverted();
 5603   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5604   emit_int16(0x6C, (0xC0 | encode));
 5605 }
 5606 
 5607 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5608   evpunpcklqdq(dst, k0, src1, src2, false, vector_len);
 5609 }
 5610 
 5611 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5612   assert(VM_Version::supports_evex(), "requires AVX512F");
 5613   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 5614   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5615   attributes.set_is_evex_instruction();
 5616   attributes.set_embedded_opmask_register_specifier(mask);
 5617   if (merge) {
 5618     attributes.reset_is_clear_context();
 5619   }
 5620 
 5621   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5622   emit_int16(0x6C, (0xC0 | encode));
 5623 }
 5624 
 5625 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 5626   evpunpckhqdq(dst, k0, src1, src2, false, vector_len);
 5627 }
 5628 
 5629 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
 5630   assert(VM_Version::supports_evex(), "requires AVX512F");
 5631   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 5632   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 5633   attributes.set_is_evex_instruction();
 5634   attributes.set_embedded_opmask_register_specifier(mask);
 5635   if (merge) {
 5636     attributes.reset_is_clear_context();
 5637   }
 5638 
 5639   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 5640   emit_int16(0x6D, (0xC0 | encode));
 5641 }
 5642 
 5643 void Assembler::push(int32_t imm32) {
 5644   // in 64bits we push 64bits onto the stack but only
 5645   // take a 32bit immediate
 5646   emit_int8(0x68);
 5647   emit_int32(imm32);
 5648 }
 5649 
 5650 void Assembler::push(Register src) {
 5651   int encode = prefix_and_encode(src->encoding());
 5652   emit_int8(0x50 | encode);
 5653 }
 5654 
 5655 void Assembler::pushf() {
 5656   emit_int8((unsigned char)0x9C);
 5657 }
 5658 
 5659 #ifndef _LP64 // no 32bit push/pop on amd64
 5660 void Assembler::pushl(Address src) {
 5661   // Note this will push 64bit on 64bit
 5662   InstructionMark im(this);
 5663   prefix(src);
 5664   emit_int8((unsigned char)0xFF);
 5665   emit_operand(rsi, src, 0);
 5666 }
 5667 #endif
 5668 
 5669 void Assembler::rcll(Register dst, int imm8) {
 5670   assert(isShiftCount(imm8), "illegal shift count");
 5671   int encode = prefix_and_encode(dst->encoding());
 5672   if (imm8 == 1) {
 5673     emit_int16((unsigned char)0xD1, (0xD0 | encode));
 5674   } else {
 5675     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
 5676   }
 5677 }
 5678 
 5679 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
 5680   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5681   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5682   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 5683   emit_int16(0x53, (0xC0 | encode));
 5684 }
 5685 
 5686 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
 5687   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 5688   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5689   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 5690   emit_int16(0x53, (0xC0 | encode));
 5691 }
 5692 
 5693 void Assembler::rdtsc() {
 5694   emit_int16(0x0F, 0x31);
 5695 }
 5696 
 5697 // copies data from [esi] to [edi] using rcx pointer sized words
 5698 // generic
 5699 void Assembler::rep_mov() {
 5700   // REP
 5701   // MOVSQ
 5702   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
 5703   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xA5);)
 5704 }
 5705 
 5706 // sets rcx bytes with rax, value at [edi]
 5707 void Assembler::rep_stosb() {
 5708   // REP
 5709   // STOSB
 5710   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
 5711   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAA);)
 5712 }
 5713 
 5714 // sets rcx pointer sized words with rax, value at [edi]
 5715 // generic
 5716 void Assembler::rep_stos() {
 5717   // REP
 5718   // LP64:STOSQ, LP32:STOSD
 5719   LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
 5720   NOT_LP64( emit_int16((unsigned char)0xF3,        (unsigned char)0xAB);)
 5721 }
 5722 
 5723 // scans rcx pointer sized words at [edi] for occurrence of rax,
 5724 // generic
 5725 void Assembler::repne_scan() { // repne_scan
 5726   // SCASQ
 5727   LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
 5728   NOT_LP64( emit_int16((unsigned char)0xF2,        (unsigned char)0xAF);)
 5729 }
 5730 
 5731 #ifdef _LP64
 5732 // scans rcx 4 byte words at [edi] for occurrence of rax,
 5733 // generic
 5734 void Assembler::repne_scanl() { // repne_scan
 5735   // SCASL
 5736   emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
 5737 }
 5738 #endif
 5739 
 5740 void Assembler::ret(int imm16) {
 5741   if (imm16 == 0) {
 5742     emit_int8((unsigned char)0xC3);
 5743   } else {
 5744     emit_int8((unsigned char)0xC2);
 5745     emit_int16(imm16);
 5746   }
 5747 }
 5748 
 5749 void Assembler::roll(Register dst, int imm8) {
 5750   assert(isShiftCount(imm8), "illegal shift count");
 5751   int encode = prefix_and_encode(dst->encoding());
 5752   if (imm8 == 1) {
 5753     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5754   } else {
 5755     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5756   }
 5757 }
 5758 
 5759 void Assembler::roll(Register dst) {
 5760   int encode = prefix_and_encode(dst->encoding());
 5761   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5762 }
 5763 
 5764 void Assembler::rorl(Register dst, int imm8) {
 5765   assert(isShiftCount(imm8), "illegal shift count");
 5766   int encode = prefix_and_encode(dst->encoding());
 5767   if (imm8 == 1) {
 5768     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5769   } else {
 5770     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5771   }
 5772 }
 5773 
 5774 void Assembler::rorl(Register dst) {
 5775   int encode = prefix_and_encode(dst->encoding());
 5776   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5777 }
 5778 
 5779 #ifdef _LP64
 5780 void Assembler::rorq(Register dst) {
 5781   int encode = prefixq_and_encode(dst->encoding());
 5782   emit_int16((unsigned char)0xD3, (0xC8 | encode));
 5783 }
 5784 
 5785 void Assembler::rorq(Register dst, int imm8) {
 5786   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5787   int encode = prefixq_and_encode(dst->encoding());
 5788   if (imm8 == 1) {
 5789     emit_int16((unsigned char)0xD1, (0xC8 | encode));
 5790   } else {
 5791     emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
 5792   }
 5793 }
 5794 
 5795 void Assembler::rolq(Register dst) {
 5796   int encode = prefixq_and_encode(dst->encoding());
 5797   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 5798 }
 5799 
 5800 void Assembler::rolq(Register dst, int imm8) {
 5801   assert(isShiftCount(imm8 >> 1), "illegal shift count");
 5802   int encode = prefixq_and_encode(dst->encoding());
 5803   if (imm8 == 1) {
 5804     emit_int16((unsigned char)0xD1, (0xC0 | encode));
 5805   } else {
 5806     emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
 5807   }
 5808 }
 5809 #endif
 5810 
 5811 void Assembler::sahf() {
 5812 #ifdef _LP64
 5813   // Not supported in 64bit mode
 5814   ShouldNotReachHere();
 5815 #endif
 5816   emit_int8((unsigned char)0x9E);
 5817 }
 5818 
 5819 void Assembler::sall(Address dst, int imm8) {
 5820   InstructionMark im(this);
 5821   assert(isShiftCount(imm8), "illegal shift count");
 5822   prefix(dst);
 5823   if (imm8 == 1) {
 5824     emit_int8((unsigned char)0xD1);
 5825     emit_operand(as_Register(4), dst, 0);
 5826   }
 5827   else {
 5828     emit_int8((unsigned char)0xC1);
 5829     emit_operand(as_Register(4), dst, 1);
 5830     emit_int8(imm8);
 5831   }
 5832 }
 5833 
 5834 void Assembler::sall(Address dst) {
 5835   InstructionMark im(this);
 5836   prefix(dst);
 5837   emit_int8((unsigned char)0xD3);
 5838   emit_operand(as_Register(4), dst, 0);
 5839 }
 5840 
 5841 void Assembler::sall(Register dst, int imm8) {
 5842   assert(isShiftCount(imm8), "illegal shift count");
 5843   int encode = prefix_and_encode(dst->encoding());
 5844   if (imm8 == 1) {
 5845     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 5846   } else {
 5847     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 5848   }
 5849 }
 5850 
 5851 void Assembler::sall(Register dst) {
 5852   int encode = prefix_and_encode(dst->encoding());
 5853   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 5854 }
 5855 
 5856 void Assembler::sarl(Address dst, int imm8) {
 5857   assert(isShiftCount(imm8), "illegal shift count");
 5858   InstructionMark im(this);
 5859   prefix(dst);
 5860   if (imm8 == 1) {
 5861     emit_int8((unsigned char)0xD1);
 5862     emit_operand(as_Register(7), dst, 0);
 5863   }
 5864   else {
 5865     emit_int8((unsigned char)0xC1);
 5866     emit_operand(as_Register(7), dst, 1);
 5867     emit_int8(imm8);
 5868   }
 5869 }
 5870 
 5871 void Assembler::sarl(Address dst) {
 5872   InstructionMark im(this);
 5873   prefix(dst);
 5874   emit_int8((unsigned char)0xD3);
 5875   emit_operand(as_Register(7), dst, 0);
 5876 }
 5877 
 5878 void Assembler::sarl(Register dst, int imm8) {
 5879   int encode = prefix_and_encode(dst->encoding());
 5880   assert(isShiftCount(imm8), "illegal shift count");
 5881   if (imm8 == 1) {
 5882     emit_int16((unsigned char)0xD1, (0xF8 | encode));
 5883   } else {
 5884     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
 5885   }
 5886 }
 5887 
 5888 void Assembler::sarl(Register dst) {
 5889   int encode = prefix_and_encode(dst->encoding());
 5890   emit_int16((unsigned char)0xD3, (0xF8 | encode));
 5891 }
 5892 
 5893 void Assembler::sbbl(Address dst, int32_t imm32) {
 5894   InstructionMark im(this);
 5895   prefix(dst);
 5896   emit_arith_operand(0x81, rbx, dst, imm32);
 5897 }
 5898 
 5899 void Assembler::sbbl(Register dst, int32_t imm32) {
 5900   prefix(dst);
 5901   emit_arith(0x81, 0xD8, dst, imm32);
 5902 }
 5903 
 5904 
 5905 void Assembler::sbbl(Register dst, Address src) {
 5906   InstructionMark im(this);
 5907   prefix(src, dst);
 5908   emit_int8(0x1B);
 5909   emit_operand(dst, src, 0);
 5910 }
 5911 
 5912 void Assembler::sbbl(Register dst, Register src) {
 5913   (void) prefix_and_encode(dst->encoding(), src->encoding());
 5914   emit_arith(0x1B, 0xC0, dst, src);
 5915 }
 5916 
 5917 void Assembler::setb(Condition cc, Register dst) {
 5918   assert(0 <= cc && cc < 16, "illegal cc");
 5919   int encode = prefix_and_encode(dst->encoding(), true);
 5920   emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
 5921 }
 5922 
 5923 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
 5924   assert(VM_Version::supports_ssse3(), "");
 5925   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5926   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5927   emit_int24(0x0F, (0xC0 | encode), imm8);
 5928 }
 5929 
 5930 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
 5931   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
 5932          vector_len == AVX_256bit? VM_Version::supports_avx2() :
 5933          0, "");
 5934   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 5935   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5936   emit_int24(0x0F, (0xC0 | encode), imm8);
 5937 }
 5938 
 5939 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 5940   assert(VM_Version::supports_evex(), "");
 5941   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 5942   attributes.set_is_evex_instruction();
 5943   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5944   emit_int24(0x3, (0xC0 | encode), imm8);
 5945 }
 5946 
 5947 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
 5948   assert(VM_Version::supports_sse4_1(), "");
 5949   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 5950   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 5951   emit_int24(0x0E, (0xC0 | encode), imm8);
 5952 }
 5953 
 5954 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
 5955   assert(VM_Version::supports_sha(), "");
 5956   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
 5957   emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
 5958 }
 5959 
 5960 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
 5961   assert(VM_Version::supports_sha(), "");
 5962   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5963   emit_int16((unsigned char)0xC8, (0xC0 | encode));
 5964 }
 5965 
 5966 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
 5967   assert(VM_Version::supports_sha(), "");
 5968   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5969   emit_int16((unsigned char)0xC9, (0xC0 | encode));
 5970 }
 5971 
 5972 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
 5973   assert(VM_Version::supports_sha(), "");
 5974   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5975   emit_int16((unsigned char)0xCA, (0xC0 | encode));
 5976 }
 5977 
 5978 // xmm0 is implicit additional source to this instruction.
 5979 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
 5980   assert(VM_Version::supports_sha(), "");
 5981   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5982   emit_int16((unsigned char)0xCB, (0xC0 | encode));
 5983 }
 5984 
 5985 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
 5986   assert(VM_Version::supports_sha(), "");
 5987   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5988   emit_int16((unsigned char)0xCC, (0xC0 | encode));
 5989 }
 5990 
 5991 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
 5992   assert(VM_Version::supports_sha(), "");
 5993   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
 5994   emit_int16((unsigned char)0xCD, (0xC0 | encode));
 5995 }
 5996 
 5997 
 5998 void Assembler::shll(Register dst, int imm8) {
 5999   assert(isShiftCount(imm8), "illegal shift count");
 6000   int encode = prefix_and_encode(dst->encoding());
 6001   if (imm8 == 1 ) {
 6002     emit_int16((unsigned char)0xD1, (0xE0 | encode));
 6003   } else {
 6004     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
 6005   }
 6006 }
 6007 
 6008 void Assembler::shll(Register dst) {
 6009   int encode = prefix_and_encode(dst->encoding());
 6010   emit_int16((unsigned char)0xD3, (0xE0 | encode));
 6011 }
 6012 
 6013 void Assembler::shrl(Register dst, int imm8) {
 6014   assert(isShiftCount(imm8), "illegal shift count");
 6015   int encode = prefix_and_encode(dst->encoding());
 6016   if (imm8 == 1) {
 6017     emit_int16((unsigned char)0xD1, (0xE8 | encode));
 6018   }
 6019   else {
 6020     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
 6021   }
 6022 }
 6023 
 6024 void Assembler::shrl(Register dst) {
 6025   int encode = prefix_and_encode(dst->encoding());
 6026   emit_int16((unsigned char)0xD3, (0xE8 | encode));
 6027 }
 6028 
 6029 void Assembler::shrl(Address dst) {
 6030   InstructionMark im(this);
 6031   prefix(dst);
 6032   emit_int8((unsigned char)0xD3);
 6033   emit_operand(as_Register(5), dst, 0);
 6034 }
 6035 
 6036 void Assembler::shrl(Address dst, int imm8) {
 6037   InstructionMark im(this);
 6038   assert(isShiftCount(imm8), "illegal shift count");
 6039   prefix(dst);
 6040   if (imm8 == 1) {
 6041     emit_int8((unsigned char)0xD1);
 6042     emit_operand(as_Register(5), dst, 0);
 6043   }
 6044   else {
 6045     emit_int8((unsigned char)0xC1);
 6046     emit_operand(as_Register(5), dst, 1);
 6047     emit_int8(imm8);
 6048   }
 6049 }
 6050 
 6051 
 6052 void Assembler::shldl(Register dst, Register src) {
 6053   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6054   emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
 6055 }
 6056 
 6057 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
 6058   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6059   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 6060 }
 6061 
 6062 void Assembler::shrdl(Register dst, Register src) {
 6063   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6064   emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
 6065 }
 6066 
 6067 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
 6068   int encode = prefix_and_encode(src->encoding(), dst->encoding());
 6069   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 6070 }
 6071 
 6072 #ifdef _LP64
 6073 void Assembler::shldq(Register dst, Register src, int8_t imm8) {
 6074   int encode = prefixq_and_encode(src->encoding(), dst->encoding());
 6075   emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
 6076 }
 6077 
 6078 void Assembler::shrdq(Register dst, Register src, int8_t imm8) {
 6079   int encode = prefixq_and_encode(src->encoding(), dst->encoding());
 6080   emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
 6081 }
 6082 #endif
 6083 
 6084 // copies a single word from [esi] to [edi]
 6085 void Assembler::smovl() {
 6086   emit_int8((unsigned char)0xA5);
 6087 }
 6088 
 6089 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
 6090   assert(VM_Version::supports_sse4_1(), "");
 6091   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6092   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6093   emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
 6094 }
 6095 
 6096 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
 6097   assert(VM_Version::supports_sse4_1(), "");
 6098   InstructionMark im(this);
 6099   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6100   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6101   emit_int8(0x0B);
 6102   emit_operand(dst, src, 1);
 6103   emit_int8((unsigned char)rmode);
 6104 }
 6105 
 6106 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 6107   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6108   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6109   attributes.set_rex_vex_w_reverted();
 6110   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6111   emit_int16(0x51, (0xC0 | encode));
 6112 }
 6113 
 6114 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 6115   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6116   InstructionMark im(this);
 6117   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6118   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6119   attributes.set_rex_vex_w_reverted();
 6120   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6121   emit_int8(0x51);
 6122   emit_operand(dst, src, 0);
 6123 }
 6124 
 6125 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 6126   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6127   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6128   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6129   emit_int16(0x51, (0xC0 | encode));
 6130 }
 6131 
 6132 void Assembler::std() {
 6133   emit_int8((unsigned char)0xFD);
 6134 }
 6135 
 6136 void Assembler::sqrtss(XMMRegister dst, Address src) {
 6137   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6138   InstructionMark im(this);
 6139   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6140   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6141   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6142   emit_int8(0x51);
 6143   emit_operand(dst, src, 0);
 6144 }
 6145 
 6146 void Assembler::stmxcsr( Address dst) {
 6147   if (UseAVX > 0 ) {
 6148     assert(VM_Version::supports_avx(), "");
 6149     InstructionMark im(this);
 6150     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6151     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6152     emit_int8((unsigned char)0xAE);
 6153     emit_operand(as_Register(3), dst, 0);
 6154   } else {
 6155     NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6156     InstructionMark im(this);
 6157     prefix(dst);
 6158     emit_int16(0x0F, (unsigned char)0xAE);
 6159     emit_operand(as_Register(3), dst, 0);
 6160   }
 6161 }
 6162 
 6163 void Assembler::subl(Address dst, int32_t imm32) {
 6164   InstructionMark im(this);
 6165   prefix(dst);
 6166   emit_arith_operand(0x81, rbp, dst, imm32);
 6167 }
 6168 
 6169 void Assembler::subl(Address dst, Register src) {
 6170   InstructionMark im(this);
 6171   prefix(dst, src);
 6172   emit_int8(0x29);
 6173   emit_operand(src, dst, 0);
 6174 }
 6175 
 6176 void Assembler::subl(Register dst, int32_t imm32) {
 6177   prefix(dst);
 6178   emit_arith(0x81, 0xE8, dst, imm32);
 6179 }
 6180 
 6181 // Force generation of a 4 byte immediate value even if it fits into 8bit
 6182 void Assembler::subl_imm32(Register dst, int32_t imm32) {
 6183   prefix(dst);
 6184   emit_arith_imm32(0x81, 0xE8, dst, imm32);
 6185 }
 6186 
 6187 void Assembler::subl(Register dst, Address src) {
 6188   InstructionMark im(this);
 6189   prefix(src, dst);
 6190   emit_int8(0x2B);
 6191   emit_operand(dst, src, 0);
 6192 }
 6193 
 6194 void Assembler::subl(Register dst, Register src) {
 6195   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6196   emit_arith(0x2B, 0xC0, dst, src);
 6197 }
 6198 
 6199 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 6200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6201   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6202   attributes.set_rex_vex_w_reverted();
 6203   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6204   emit_int16(0x5C, (0xC0 | encode));
 6205 }
 6206 
 6207 void Assembler::subsd(XMMRegister dst, Address src) {
 6208   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6209   InstructionMark im(this);
 6210   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6211   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6212   attributes.set_rex_vex_w_reverted();
 6213   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6214   emit_int8(0x5C);
 6215   emit_operand(dst, src, 0);
 6216 }
 6217 
 6218 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 6219   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6220   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
 6221   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6222   emit_int16(0x5C, (0xC0 | encode));
 6223 }
 6224 
 6225 void Assembler::subss(XMMRegister dst, Address src) {
 6226   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6227   InstructionMark im(this);
 6228   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6229   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6230   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6231   emit_int8(0x5C);
 6232   emit_operand(dst, src, 0);
 6233 }
 6234 
 6235 void Assembler::testb(Register dst, int imm8) {
 6236   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 6237   if (dst == rax) {
 6238     emit_int8((unsigned char)0xA8);
 6239     emit_int8(imm8);
 6240   } else {
 6241     (void) prefix_and_encode(dst->encoding(), true);
 6242     emit_arith_b(0xF6, 0xC0, dst, imm8);
 6243   }
 6244 }
 6245 
 6246 void Assembler::testb(Address dst, int imm8) {
 6247   InstructionMark im(this);
 6248   prefix(dst);
 6249   emit_int8((unsigned char)0xF6);
 6250   emit_operand(rax, dst, 1);
 6251   emit_int8(imm8);
 6252 }
 6253 
 6254 void Assembler::testl(Address dst, int32_t imm32) {
 6255   InstructionMark im(this);
 6256   prefix(dst);
 6257   emit_int8((unsigned char)0xF7);
 6258   emit_operand(as_Register(0), dst, 4);
 6259   emit_int32(imm32);
 6260 }
 6261 
 6262 void Assembler::testl(Register dst, int32_t imm32) {
 6263   // not using emit_arith because test
 6264   // doesn't support sign-extension of
 6265   // 8bit operands
 6266   if (dst == rax) {
 6267     emit_int8((unsigned char)0xA9);
 6268     emit_int32(imm32);
 6269   } else {
 6270     int encode = dst->encoding();
 6271     encode = prefix_and_encode(encode);
 6272     emit_int16((unsigned char)0xF7, (0xC0 | encode));
 6273     emit_int32(imm32);
 6274   }
 6275 }
 6276 
 6277 void Assembler::testl(Register dst, Register src) {
 6278   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6279   emit_arith(0x85, 0xC0, dst, src);
 6280 }
 6281 
 6282 void Assembler::testl(Register dst, Address src) {
 6283   InstructionMark im(this);
 6284   prefix(src, dst);
 6285   emit_int8((unsigned char)0x85);
 6286   emit_operand(dst, src, 0);
 6287 }
 6288 
 6289 void Assembler::tzcntl(Register dst, Register src) {
 6290   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6291   emit_int8((unsigned char)0xF3);
 6292   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6293   emit_int24(0x0F,
 6294              (unsigned char)0xBC,
 6295              0xC0 | encode);
 6296 }
 6297 
 6298 void Assembler::tzcntl(Register dst, Address src) {
 6299   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6300   InstructionMark im(this);
 6301   emit_int8((unsigned char)0xF3);
 6302   prefix(src, dst);
 6303   emit_int16(0x0F, (unsigned char)0xBC);
 6304   emit_operand(dst, src, 0);
 6305 }
 6306 
 6307 void Assembler::tzcntq(Register dst, Register src) {
 6308   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6309   emit_int8((unsigned char)0xF3);
 6310   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
 6311   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
 6312 }
 6313 
 6314 void Assembler::tzcntq(Register dst, Address src) {
 6315   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
 6316   InstructionMark im(this);
 6317   emit_int8((unsigned char)0xF3);
 6318   prefixq(src, dst);
 6319   emit_int16(0x0F, (unsigned char)0xBC);
 6320   emit_operand(dst, src, 0);
 6321 }
 6322 
 6323 void Assembler::ucomisd(XMMRegister dst, Address src) {
 6324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6325   InstructionMark im(this);
 6326   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6327   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6328   attributes.set_rex_vex_w_reverted();
 6329   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6330   emit_int8(0x2E);
 6331   emit_operand(dst, src, 0);
 6332 }
 6333 
 6334 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 6335   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6336   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6337   attributes.set_rex_vex_w_reverted();
 6338   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6339   emit_int16(0x2E, (0xC0 | encode));
 6340 }
 6341 
 6342 void Assembler::ucomiss(XMMRegister dst, Address src) {
 6343   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6344   InstructionMark im(this);
 6345   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6346   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6347   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6348   emit_int8(0x2E);
 6349   emit_operand(dst, src, 0);
 6350 }
 6351 
 6352 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 6353   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 6354   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6355   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6356   emit_int16(0x2E, (0xC0 | encode));
 6357 }
 6358 
 6359 void Assembler::xabort(int8_t imm8) {
 6360   emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
 6361 }
 6362 
 6363 void Assembler::xaddb(Address dst, Register src) {
 6364   InstructionMark im(this);
 6365   prefix(dst, src, true);
 6366   emit_int16(0x0F, (unsigned char)0xC0);
 6367   emit_operand(src, dst, 0);
 6368 }
 6369 
 6370 void Assembler::xaddw(Address dst, Register src) {
 6371   InstructionMark im(this);
 6372   emit_int8(0x66);
 6373   prefix(dst, src);
 6374   emit_int16(0x0F, (unsigned char)0xC1);
 6375   emit_operand(src, dst, 0);
 6376 }
 6377 
 6378 void Assembler::xaddl(Address dst, Register src) {
 6379   InstructionMark im(this);
 6380   prefix(dst, src);
 6381   emit_int16(0x0F, (unsigned char)0xC1);
 6382   emit_operand(src, dst, 0);
 6383 }
 6384 
 6385 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
 6386   InstructionMark im(this);
 6387   relocate(rtype);
 6388   if (abort.is_bound()) {
 6389     address entry = target(abort);
 6390     assert(entry != nullptr, "abort entry null");
 6391     int offset = checked_cast<int>(entry - pc());
 6392     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6393     emit_int32(offset - 6); // 2 opcode + 4 address
 6394   } else {
 6395     abort.add_patch_at(code(), locator());
 6396     emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
 6397     emit_int32(0);
 6398   }
 6399 }
 6400 
 6401 void Assembler::xchgb(Register dst, Address src) { // xchg
 6402   InstructionMark im(this);
 6403   prefix(src, dst, true);
 6404   emit_int8((unsigned char)0x86);
 6405   emit_operand(dst, src, 0);
 6406 }
 6407 
 6408 void Assembler::xchgw(Register dst, Address src) { // xchg
 6409   InstructionMark im(this);
 6410   emit_int8(0x66);
 6411   prefix(src, dst);
 6412   emit_int8((unsigned char)0x87);
 6413   emit_operand(dst, src, 0);
 6414 }
 6415 
 6416 void Assembler::xchgl(Register dst, Address src) { // xchg
 6417   InstructionMark im(this);
 6418   prefix(src, dst);
 6419   emit_int8((unsigned char)0x87);
 6420   emit_operand(dst, src, 0);
 6421 }
 6422 
 6423 void Assembler::xchgl(Register dst, Register src) {
 6424   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 6425   emit_int16((unsigned char)0x87, (0xC0 | encode));
 6426 }
 6427 
 6428 void Assembler::xend() {
 6429   emit_int24(0x0F, 0x01, (unsigned char)0xD5);
 6430 }
 6431 
 6432 void Assembler::xgetbv() {
 6433   emit_int24(0x0F, 0x01, (unsigned char)0xD0);
 6434 }
 6435 
 6436 void Assembler::xorl(Address dst, int32_t imm32) {
 6437   InstructionMark im(this);
 6438   prefix(dst);
 6439   emit_arith_operand(0x81, as_Register(6), dst, imm32);
 6440 }
 6441 
 6442 void Assembler::xorl(Register dst, int32_t imm32) {
 6443   prefix(dst);
 6444   emit_arith(0x81, 0xF0, dst, imm32);
 6445 }
 6446 
 6447 void Assembler::xorl(Register dst, Address src) {
 6448   InstructionMark im(this);
 6449   prefix(src, dst);
 6450   emit_int8(0x33);
 6451   emit_operand(dst, src, 0);
 6452 }
 6453 
 6454 void Assembler::xorl(Register dst, Register src) {
 6455   (void) prefix_and_encode(dst->encoding(), src->encoding());
 6456   emit_arith(0x33, 0xC0, dst, src);
 6457 }
 6458 
 6459 void Assembler::xorl(Address dst, Register src) {
 6460   InstructionMark im(this);
 6461   prefix(dst, src);
 6462   emit_int8(0x31);
 6463   emit_operand(src, dst, 0);
 6464 }
 6465 
 6466 void Assembler::xorb(Register dst, Address src) {
 6467   InstructionMark im(this);
 6468   prefix(src, dst);
 6469   emit_int8(0x32);
 6470   emit_operand(dst, src, 0);
 6471 }
 6472 
 6473 void Assembler::xorb(Address dst, Register src) {
 6474   InstructionMark im(this);
 6475   prefix(dst, src, true);
 6476   emit_int8(0x30);
 6477   emit_operand(src, dst, 0);
 6478 }
 6479 
 6480 void Assembler::xorw(Register dst, Register src) {
 6481   (void)prefix_and_encode(dst->encoding(), src->encoding());
 6482   emit_arith(0x33, 0xC0, dst, src);
 6483 }
 6484 
 6485 // AVX 3-operands scalar float-point arithmetic instructions
 6486 
 6487 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
 6488   assert(VM_Version::supports_avx(), "");
 6489   InstructionMark im(this);
 6490   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6491   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6492   attributes.set_rex_vex_w_reverted();
 6493   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6494   emit_int8(0x58);
 6495   emit_operand(dst, src, 0);
 6496 }
 6497 
 6498 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6499   assert(VM_Version::supports_avx(), "");
 6500   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6501   attributes.set_rex_vex_w_reverted();
 6502   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6503   emit_int16(0x58, (0xC0 | encode));
 6504 }
 6505 
 6506 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
 6507   assert(VM_Version::supports_avx(), "");
 6508   InstructionMark im(this);
 6509   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6510   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6511   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6512   emit_int8(0x58);
 6513   emit_operand(dst, src, 0);
 6514 }
 6515 
 6516 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6517   assert(VM_Version::supports_avx(), "");
 6518   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6519   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6520   emit_int16(0x58, (0xC0 | encode));
 6521 }
 6522 
 6523 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
 6524   assert(VM_Version::supports_avx(), "");
 6525   InstructionMark im(this);
 6526   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6527   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6528   attributes.set_rex_vex_w_reverted();
 6529   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6530   emit_int8(0x5E);
 6531   emit_operand(dst, src, 0);
 6532 }
 6533 
 6534 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6535   assert(VM_Version::supports_avx(), "");
 6536   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6537   attributes.set_rex_vex_w_reverted();
 6538   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6539   emit_int16(0x5E, (0xC0 | encode));
 6540 }
 6541 
 6542 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
 6543   assert(VM_Version::supports_avx(), "");
 6544   InstructionMark im(this);
 6545   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6547   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6548   emit_int8(0x5E);
 6549   emit_operand(dst, src, 0);
 6550 }
 6551 
 6552 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6553   assert(VM_Version::supports_avx(), "");
 6554   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6555   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6556   emit_int16(0x5E, (0xC0 | encode));
 6557 }
 6558 
 6559 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6560   assert(VM_Version::supports_fma(), "");
 6561   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6562   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6563   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6564 }
 6565 
 6566 void Assembler::evfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2, EvexRoundPrefix rmode) { // Need to add rmode for rounding mode support
 6567   assert(VM_Version::supports_evex(), "");
 6568   InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6569   attributes.set_extended_context();
 6570   attributes.set_is_evex_instruction();
 6571   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6572   emit_int16((unsigned char)0xAD, (0xC0 | encode));
 6573 }
 6574 
 6575 void Assembler::vfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6576   assert(VM_Version::supports_fma(), "");
 6577   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6578   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6579   emit_int16((unsigned char)0xAD, (0xC0 | encode));
 6580 }
 6581 
 6582 void Assembler::vfnmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6583   assert(VM_Version::supports_fma(), "");
 6584   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6585   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6586   emit_int16((unsigned char)0xBD, (0xC0 | encode));
 6587 }
 6588 
 6589 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
 6590   assert(VM_Version::supports_fma(), "");
 6591   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6592   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6593   emit_int16((unsigned char)0xB9, (0xC0 | encode));
 6594 }
 6595 
 6596 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
 6597   assert(VM_Version::supports_avx(), "");
 6598   InstructionMark im(this);
 6599   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6600   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6601   attributes.set_rex_vex_w_reverted();
 6602   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6603   emit_int8(0x59);
 6604   emit_operand(dst, src, 0);
 6605 }
 6606 
 6607 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6608   assert(VM_Version::supports_avx(), "");
 6609   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6610   attributes.set_rex_vex_w_reverted();
 6611   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6612   emit_int16(0x59, (0xC0 | encode));
 6613 }
 6614 
 6615 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
 6616   assert(VM_Version::supports_avx(), "");
 6617   InstructionMark im(this);
 6618   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6620   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6621   emit_int8(0x59);
 6622   emit_operand(dst, src, 0);
 6623 }
 6624 
 6625 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6626   assert(VM_Version::supports_avx(), "");
 6627   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6628   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6629   emit_int16(0x59, (0xC0 | encode));
 6630 }
 6631 
 6632 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
 6633   assert(VM_Version::supports_avx(), "");
 6634   InstructionMark im(this);
 6635   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6636   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
 6637   attributes.set_rex_vex_w_reverted();
 6638   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6639   emit_int8(0x5C);
 6640   emit_operand(dst, src, 0);
 6641 }
 6642 
 6643 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6644   assert(VM_Version::supports_avx(), "");
 6645   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6646   attributes.set_rex_vex_w_reverted();
 6647   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 6648   emit_int16(0x5C, (0xC0 | encode));
 6649 }
 6650 
 6651 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
 6652   assert(VM_Version::supports_avx(), "");
 6653   InstructionMark im(this);
 6654   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6655   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
 6656   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6657   emit_int8(0x5C);
 6658   emit_operand(dst, src, 0);
 6659 }
 6660 
 6661 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 6662   assert(VM_Version::supports_avx(), "");
 6663   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6664   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
 6665   emit_int16(0x5C, (0xC0 | encode));
 6666 }
 6667 
 6668 //====================VECTOR ARITHMETIC=====================================
 6669 
 6670 // Float-point vector arithmetic
 6671 
 6672 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
 6673   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6674   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6675   attributes.set_rex_vex_w_reverted();
 6676   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6677   emit_int16(0x58, (0xC0 | encode));
 6678 }
 6679 
 6680 void Assembler::addpd(XMMRegister dst, Address src) {
 6681   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6682   InstructionMark im(this);
 6683   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6684   attributes.set_rex_vex_w_reverted();
 6685   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6686   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6687   emit_int8(0x58);
 6688   emit_operand(dst, src, 0);
 6689 }
 6690 
 6691 
 6692 void Assembler::addps(XMMRegister dst, XMMRegister src) {
 6693   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6694   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6695   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6696   emit_int16(0x58, (0xC0 | encode));
 6697 }
 6698 
 6699 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6700   assert(VM_Version::supports_avx(), "");
 6701   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6702   attributes.set_rex_vex_w_reverted();
 6703   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6704   emit_int16(0x58, (0xC0 | encode));
 6705 }
 6706 
 6707 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6708   assert(VM_Version::supports_avx(), "");
 6709   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6710   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6711   emit_int16(0x58, (0xC0 | encode));
 6712 }
 6713 
 6714 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6715   assert(VM_Version::supports_avx(), "");
 6716   InstructionMark im(this);
 6717   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6718   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6719   attributes.set_rex_vex_w_reverted();
 6720   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6721   emit_int8(0x58);
 6722   emit_operand(dst, src, 0);
 6723 }
 6724 
 6725 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6726   assert(VM_Version::supports_avx(), "");
 6727   InstructionMark im(this);
 6728   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6729   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6730   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6731   emit_int8(0x58);
 6732   emit_operand(dst, src, 0);
 6733 }
 6734 
 6735 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
 6736   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6737   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6738   attributes.set_rex_vex_w_reverted();
 6739   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6740   emit_int16(0x5C, (0xC0 | encode));
 6741 }
 6742 
 6743 void Assembler::subps(XMMRegister dst, XMMRegister src) {
 6744   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6745   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6746   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6747   emit_int16(0x5C, (0xC0 | encode));
 6748 }
 6749 
 6750 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6751   assert(VM_Version::supports_avx(), "");
 6752   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6753   attributes.set_rex_vex_w_reverted();
 6754   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6755   emit_int16(0x5C, (0xC0 | encode));
 6756 }
 6757 
 6758 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6759   assert(VM_Version::supports_avx(), "");
 6760   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6761   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6762   emit_int16(0x5C, (0xC0 | encode));
 6763 }
 6764 
 6765 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6766   assert(VM_Version::supports_avx(), "");
 6767   InstructionMark im(this);
 6768   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6769   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6770   attributes.set_rex_vex_w_reverted();
 6771   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6772   emit_int8(0x5C);
 6773   emit_operand(dst, src, 0);
 6774 }
 6775 
 6776 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6777   assert(VM_Version::supports_avx(), "");
 6778   InstructionMark im(this);
 6779   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6780   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6781   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6782   emit_int8(0x5C);
 6783   emit_operand(dst, src, 0);
 6784 }
 6785 
 6786 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
 6787   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6788   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6789   attributes.set_rex_vex_w_reverted();
 6790   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6791   emit_int16(0x59, (0xC0 | encode));
 6792 }
 6793 
 6794 void Assembler::mulpd(XMMRegister dst, Address src) {
 6795   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6796   InstructionMark im(this);
 6797   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6798   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6799   attributes.set_rex_vex_w_reverted();
 6800   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6801   emit_int8(0x59);
 6802   emit_operand(dst, src, 0);
 6803 }
 6804 
 6805 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
 6806   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6807   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6808   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6809   emit_int16(0x59, (0xC0 | encode));
 6810 }
 6811 
 6812 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6813   assert(VM_Version::supports_avx(), "");
 6814   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6815   attributes.set_rex_vex_w_reverted();
 6816   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6817   emit_int16(0x59, (0xC0 | encode));
 6818 }
 6819 
 6820 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6821   assert(VM_Version::supports_avx(), "");
 6822   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6824   emit_int16(0x59, (0xC0 | encode));
 6825 }
 6826 
 6827 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6828   assert(VM_Version::supports_avx(), "");
 6829   InstructionMark im(this);
 6830   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6832   attributes.set_rex_vex_w_reverted();
 6833   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6834   emit_int8(0x59);
 6835   emit_operand(dst, src, 0);
 6836 }
 6837 
 6838 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6839   assert(VM_Version::supports_avx(), "");
 6840   InstructionMark im(this);
 6841   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6842   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6843   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6844   emit_int8(0x59);
 6845   emit_operand(dst, src, 0);
 6846 }
 6847 
 6848 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6849   assert(VM_Version::supports_fma(), "");
 6850   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6851   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6852   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6853 }
 6854 
 6855 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
 6856   assert(VM_Version::supports_fma(), "");
 6857   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6858   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6859   emit_int16((unsigned char)0xB8, (0xC0 | encode));
 6860 }
 6861 
 6862 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6863   assert(VM_Version::supports_fma(), "");
 6864   InstructionMark im(this);
 6865   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6866   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6867   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6868   emit_int8((unsigned char)0xB8);
 6869   emit_operand(dst, src2, 0);
 6870 }
 6871 
 6872 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 6873   assert(VM_Version::supports_fma(), "");
 6874   InstructionMark im(this);
 6875   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6876   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6877   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 6878   emit_int8((unsigned char)0xB8);
 6879   emit_operand(dst, src2, 0);
 6880 }
 6881 
 6882 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
 6883   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6884   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6885   attributes.set_rex_vex_w_reverted();
 6886   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6887   emit_int16(0x5E, (0xC0 | encode));
 6888 }
 6889 
 6890 void Assembler::divps(XMMRegister dst, XMMRegister src) {
 6891   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 6892   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6893   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6894   emit_int16(0x5E, (0xC0 | encode));
 6895 }
 6896 
 6897 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6898   assert(VM_Version::supports_avx(), "");
 6899   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6900   attributes.set_rex_vex_w_reverted();
 6901   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6902   emit_int16(0x5E, (0xC0 | encode));
 6903 }
 6904 
 6905 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 6906   assert(VM_Version::supports_avx(), "");
 6907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6908   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6909   emit_int16(0x5E, (0xC0 | encode));
 6910 }
 6911 
 6912 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6913   assert(VM_Version::supports_avx(), "");
 6914   InstructionMark im(this);
 6915   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6916   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6917   attributes.set_rex_vex_w_reverted();
 6918   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6919   emit_int8(0x5E);
 6920   emit_operand(dst, src, 0);
 6921 }
 6922 
 6923 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 6924   assert(VM_Version::supports_avx(), "");
 6925   InstructionMark im(this);
 6926   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6927   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 6928   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 6929   emit_int8(0x5E);
 6930   emit_operand(dst, src, 0);
 6931 }
 6932 
 6933 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
 6934   assert(VM_Version::supports_avx(), "");
 6935   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6936   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6937   emit_int24(0x09, (0xC0 | encode), (rmode));
 6938 }
 6939 
 6940 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode,  int vector_len) {
 6941   assert(VM_Version::supports_avx(), "");
 6942   InstructionMark im(this);
 6943   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
 6944   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6945   emit_int8(0x09);
 6946   emit_operand(dst, src, 1);
 6947   emit_int8((rmode));
 6948 }
 6949 
 6950 void Assembler::vroundsd(XMMRegister dst, XMMRegister src, XMMRegister src2, int32_t rmode) {
 6951   assert(VM_Version::supports_avx(), "");
 6952   assert(rmode <= 0x0f, "rmode 0x%x", rmode);
 6953   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6954   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6955   emit_int24(0x0B, (0xC0 | encode), (rmode));
 6956 }
 6957 
 6958 void Assembler::vrndscalesd(XMMRegister dst,  XMMRegister src1, XMMRegister src2, int32_t rmode) {
 6959   assert(VM_Version::supports_evex(), "requires EVEX support");
 6960   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 6961   attributes.set_is_evex_instruction();
 6962   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6963   emit_int24(0x0B, (0xC0 | encode), (rmode));
 6964 }
 6965 
 6966 void Assembler::vrndscalepd(XMMRegister dst,  XMMRegister src,  int32_t rmode, int vector_len) {
 6967   assert(VM_Version::supports_evex(), "requires EVEX support");
 6968   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6969   attributes.set_is_evex_instruction();
 6970   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6971   emit_int24(0x09, (0xC0 | encode), (rmode));
 6972 }
 6973 
 6974 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
 6975   assert(VM_Version::supports_evex(), "requires EVEX support");
 6976   assert(dst != xnoreg, "sanity");
 6977   InstructionMark im(this);
 6978   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6979   attributes.set_is_evex_instruction();
 6980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 6981   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 6982   emit_int8(0x09);
 6983   emit_operand(dst, src, 1);
 6984   emit_int8((rmode));
 6985 }
 6986 
 6987 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
 6988   assert(VM_Version::supports_avx(), "");
 6989   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6990   attributes.set_rex_vex_w_reverted();
 6991   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 6992   emit_int16(0x51, (0xC0 | encode));
 6993 }
 6994 
 6995 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
 6996   assert(VM_Version::supports_avx(), "");
 6997   InstructionMark im(this);
 6998   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 6999   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7000   attributes.set_rex_vex_w_reverted();
 7001   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7002   emit_int8(0x51);
 7003   emit_operand(dst, src, 0);
 7004 }
 7005 
 7006 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
 7007   assert(VM_Version::supports_avx(), "");
 7008   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7009   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7010   emit_int16(0x51, (0xC0 | encode));
 7011 }
 7012 
 7013 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
 7014   assert(VM_Version::supports_avx(), "");
 7015   InstructionMark im(this);
 7016   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7017   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7018   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7019   emit_int8(0x51);
 7020   emit_operand(dst, src, 0);
 7021 }
 7022 
 7023 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
 7024   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7025   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7026   attributes.set_rex_vex_w_reverted();
 7027   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7028   emit_int16(0x54, (0xC0 | encode));
 7029 }
 7030 
 7031 void Assembler::andps(XMMRegister dst, XMMRegister src) {
 7032   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7033   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7034   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7035   emit_int16(0x54, (0xC0 | encode));
 7036 }
 7037 
 7038 void Assembler::andps(XMMRegister dst, Address src) {
 7039   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7040   InstructionMark im(this);
 7041   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7042   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7043   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7044   emit_int8(0x54);
 7045   emit_operand(dst, src, 0);
 7046 }
 7047 
 7048 void Assembler::andpd(XMMRegister dst, Address src) {
 7049   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7050   InstructionMark im(this);
 7051   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7052   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7053   attributes.set_rex_vex_w_reverted();
 7054   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7055   emit_int8(0x54);
 7056   emit_operand(dst, src, 0);
 7057 }
 7058 
 7059 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7060   assert(VM_Version::supports_avx(), "");
 7061   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7062   attributes.set_rex_vex_w_reverted();
 7063   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7064   emit_int16(0x54, (0xC0 | encode));
 7065 }
 7066 
 7067 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7068   assert(VM_Version::supports_avx(), "");
 7069   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7070   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7071   emit_int16(0x54, (0xC0 | encode));
 7072 }
 7073 
 7074 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7075   assert(VM_Version::supports_avx(), "");
 7076   InstructionMark im(this);
 7077   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7078   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7079   attributes.set_rex_vex_w_reverted();
 7080   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7081   emit_int8(0x54);
 7082   emit_operand(dst, src, 0);
 7083 }
 7084 
 7085 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7086   assert(VM_Version::supports_avx(), "");
 7087   InstructionMark im(this);
 7088   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7089   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7090   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7091   emit_int8(0x54);
 7092   emit_operand(dst, src, 0);
 7093 }
 7094 
 7095 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
 7096   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7097   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7098   attributes.set_rex_vex_w_reverted();
 7099   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7100   emit_int8(0x15);
 7101   emit_int8((0xC0 | encode));
 7102 }
 7103 
 7104 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
 7105   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7106   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7107   attributes.set_rex_vex_w_reverted();
 7108   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7109   emit_int16(0x14, (0xC0 | encode));
 7110 }
 7111 
 7112 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 7113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7114   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7115   attributes.set_rex_vex_w_reverted();
 7116   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7117   emit_int16(0x57, (0xC0 | encode));
 7118 }
 7119 
 7120 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 7121   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7122   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7123   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7124   emit_int16(0x57, (0xC0 | encode));
 7125 }
 7126 
 7127 void Assembler::xorpd(XMMRegister dst, Address src) {
 7128   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7129   InstructionMark im(this);
 7130   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7131   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7132   attributes.set_rex_vex_w_reverted();
 7133   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7134   emit_int8(0x57);
 7135   emit_operand(dst, src, 0);
 7136 }
 7137 
 7138 void Assembler::xorps(XMMRegister dst, Address src) {
 7139   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7140   InstructionMark im(this);
 7141   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7142   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7143   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7144   emit_int8(0x57);
 7145   emit_operand(dst, src, 0);
 7146 }
 7147 
 7148 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7149   assert(VM_Version::supports_avx(), "");
 7150   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7151   attributes.set_rex_vex_w_reverted();
 7152   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7153   emit_int16(0x57, (0xC0 | encode));
 7154 }
 7155 
 7156 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7157   assert(VM_Version::supports_avx(), "");
 7158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7160   emit_int16(0x57, (0xC0 | encode));
 7161 }
 7162 
 7163 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7164   assert(VM_Version::supports_avx(), "");
 7165   InstructionMark im(this);
 7166   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7167   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7168   attributes.set_rex_vex_w_reverted();
 7169   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7170   emit_int8(0x57);
 7171   emit_operand(dst, src, 0);
 7172 }
 7173 
 7174 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7175   assert(VM_Version::supports_avx(), "");
 7176   InstructionMark im(this);
 7177   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7178   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7179   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7180   emit_int8(0x57);
 7181   emit_operand(dst, src, 0);
 7182 }
 7183 
 7184 // Integer vector arithmetic
 7185 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7186   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 7187          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 7188   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7189   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7190   emit_int16(0x01, (0xC0 | encode));
 7191 }
 7192 
 7193 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7194   assert(VM_Version::supports_avx() && (vector_len == 0) ||
 7195          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
 7196   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7197   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7198   emit_int16(0x02, (0xC0 | encode));
 7199 }
 7200 
 7201 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
 7202   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7203   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7204   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7205   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7206 }
 7207 
 7208 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
 7209   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7210   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7211   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7212   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7213 }
 7214 
 7215 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
 7216   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7217   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7218   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7219   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7220 }
 7221 
 7222 void Assembler::paddd(XMMRegister dst, Address src) {
 7223   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7224   InstructionMark im(this);
 7225   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7226   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7227   emit_int8((unsigned char)0xFE);
 7228   emit_operand(dst, src, 0);
 7229 }
 7230 
 7231 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
 7232   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7233   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7234   attributes.set_rex_vex_w_reverted();
 7235   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7236   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7237 }
 7238 
 7239 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
 7240   assert(VM_Version::supports_sse3(), "");
 7241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7242   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7243   emit_int16(0x01, (0xC0 | encode));
 7244 }
 7245 
 7246 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
 7247   assert(VM_Version::supports_sse3(), "");
 7248   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
 7249   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7250   emit_int16(0x02, (0xC0 | encode));
 7251 }
 7252 
 7253 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7254   assert(UseAVX > 0, "requires some form of AVX");
 7255   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7256   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7257   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 7258 }
 7259 
 7260 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7261   assert(UseAVX > 0, "requires some form of AVX");
 7262   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7263   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7264   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 7265 }
 7266 
 7267 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7268   assert(UseAVX > 0, "requires some form of AVX");
 7269   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7270   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7271   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 7272 }
 7273 
 7274 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7275   assert(UseAVX > 0, "requires some form of AVX");
 7276   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7277   attributes.set_rex_vex_w_reverted();
 7278   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7279   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 7280 }
 7281 
 7282 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7283   assert(UseAVX > 0, "requires some form of AVX");
 7284   InstructionMark im(this);
 7285   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7286   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7287   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7288   emit_int8((unsigned char)0xFC);
 7289   emit_operand(dst, src, 0);
 7290 }
 7291 
 7292 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7293   assert(UseAVX > 0, "requires some form of AVX");
 7294   InstructionMark im(this);
 7295   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7296   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7297   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7298   emit_int8((unsigned char)0xFD);
 7299   emit_operand(dst, src, 0);
 7300 }
 7301 
 7302 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7303   assert(UseAVX > 0, "requires some form of AVX");
 7304   InstructionMark im(this);
 7305   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7306   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7307   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7308   emit_int8((unsigned char)0xFE);
 7309   emit_operand(dst, src, 0);
 7310 }
 7311 
 7312 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7313   assert(UseAVX > 0, "requires some form of AVX");
 7314   InstructionMark im(this);
 7315   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7316   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7317   attributes.set_rex_vex_w_reverted();
 7318   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7319   emit_int8((unsigned char)0xD4);
 7320   emit_operand(dst, src, 0);
 7321 }
 7322 
 7323 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
 7324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7325   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7326   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7327   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7328 }
 7329 
 7330 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
 7331   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7333   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7334   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7335 }
 7336 
 7337 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
 7338   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7340   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7341 }
 7342 
 7343 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
 7344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7345   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7346   attributes.set_rex_vex_w_reverted();
 7347   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7348   emit_int8((unsigned char)0xFB);
 7349   emit_int8((0xC0 | encode));
 7350 }
 7351 
 7352 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7353   assert(UseAVX > 0, "requires some form of AVX");
 7354   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7355   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7356   emit_int16((unsigned char)0xD8, (0xC0 | encode));
 7357 }
 7358 
 7359 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7360   assert(UseAVX > 0, "requires some form of AVX");
 7361   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7362   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7363   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 7364 }
 7365 
 7366 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7367   assert(UseAVX > 0, "requires some form of AVX");
 7368   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7369   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7370   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 7371 }
 7372 
 7373 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7374   assert(UseAVX > 0, "requires some form of AVX");
 7375   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7376   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7377   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 7378 }
 7379 
 7380 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7381   assert(UseAVX > 0, "requires some form of AVX");
 7382   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7383   attributes.set_rex_vex_w_reverted();
 7384   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7385   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 7386 }
 7387 
 7388 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7389   assert(UseAVX > 0, "requires some form of AVX");
 7390   InstructionMark im(this);
 7391   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7392   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7393   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7394   emit_int8((unsigned char)0xF8);
 7395   emit_operand(dst, src, 0);
 7396 }
 7397 
 7398 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7399   assert(UseAVX > 0, "requires some form of AVX");
 7400   InstructionMark im(this);
 7401   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7402   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7403   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7404   emit_int8((unsigned char)0xF9);
 7405   emit_operand(dst, src, 0);
 7406 }
 7407 
 7408 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7409   assert(UseAVX > 0, "requires some form of AVX");
 7410   InstructionMark im(this);
 7411   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7412   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7413   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7414   emit_int8((unsigned char)0xFA);
 7415   emit_operand(dst, src, 0);
 7416 }
 7417 
 7418 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7419   assert(UseAVX > 0, "requires some form of AVX");
 7420   InstructionMark im(this);
 7421   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7422   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7423   attributes.set_rex_vex_w_reverted();
 7424   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7425   emit_int8((unsigned char)0xFB);
 7426   emit_operand(dst, src, 0);
 7427 }
 7428 
 7429 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
 7430   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7431   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7432   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7433   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7434 }
 7435 
 7436 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
 7437   assert(VM_Version::supports_sse4_1(), "");
 7438   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7439   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7440   emit_int16(0x40, (0xC0 | encode));
 7441 }
 7442 
 7443 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
 7444   assert(VM_Version::supports_sse2(), "");
 7445   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7446   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7447   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7448 }
 7449 
 7450 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7451   assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
 7452          (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
 7453          (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
 7454   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7455   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7456   emit_int16((unsigned char)0xE4, (0xC0 | encode));
 7457 }
 7458 
 7459 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7460   assert(UseAVX > 0, "requires some form of AVX");
 7461   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7462   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7463   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 7464 }
 7465 
 7466 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7467   assert(UseAVX > 0, "requires some form of AVX");
 7468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7469   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7470   emit_int16(0x40, (0xC0 | encode));
 7471 }
 7472 
 7473 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7474   assert(UseAVX > 2, "requires some form of EVEX");
 7475   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7476   attributes.set_is_evex_instruction();
 7477   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7478   emit_int16(0x40, (0xC0 | encode));
 7479 }
 7480 
 7481 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7482   assert(UseAVX > 0, "requires some form of AVX");
 7483   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7484   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7485   emit_int16((unsigned char)0xF4, (0xC0 | encode));
 7486 }
 7487 
 7488 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7489   assert(UseAVX > 0, "requires some form of AVX");
 7490   InstructionMark im(this);
 7491   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7492   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
 7493   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7494   emit_int8((unsigned char)0xD5);
 7495   emit_operand(dst, src, 0);
 7496 }
 7497 
 7498 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7499   assert(UseAVX > 0, "requires some form of AVX");
 7500   InstructionMark im(this);
 7501   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7502   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7503   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7504   emit_int8(0x40);
 7505   emit_operand(dst, src, 0);
 7506 }
 7507 
 7508 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7509   assert(UseAVX > 2, "requires some form of EVEX");
 7510   InstructionMark im(this);
 7511   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
 7512   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 7513   attributes.set_is_evex_instruction();
 7514   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7515   emit_int8(0x40);
 7516   emit_operand(dst, src, 0);
 7517 }
 7518 
 7519 // Min, max
 7520 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
 7521   assert(VM_Version::supports_sse4_1(), "");
 7522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7523   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7524   emit_int16(0x38, (0xC0 | encode));
 7525 }
 7526 
 7527 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7528   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7529         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7530   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7531   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7532   emit_int16(0x38, (0xC0 | encode));
 7533 }
 7534 
 7535 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
 7536   assert(VM_Version::supports_sse2(), "");
 7537   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7538   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7539   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7540 }
 7541 
 7542 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7543   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7544         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7547   emit_int16((unsigned char)0xEA, (0xC0 | encode));
 7548 }
 7549 
 7550 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
 7551   assert(VM_Version::supports_sse4_1(), "");
 7552   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7553   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7554   emit_int16(0x39, (0xC0 | encode));
 7555 }
 7556 
 7557 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7558   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7559         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7560   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7561   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7562   emit_int16(0x39, (0xC0 | encode));
 7563 }
 7564 
 7565 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7566   assert(UseAVX > 2, "requires AVX512F");
 7567   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7568   attributes.set_is_evex_instruction();
 7569   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7570   emit_int16(0x39, (0xC0 | encode));
 7571 }
 7572 
 7573 void Assembler::minps(XMMRegister dst, XMMRegister src) {
 7574   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7575   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7576   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7577   emit_int16(0x5D, (0xC0 | encode));
 7578 }
 7579 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7580   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7581   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7582   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7583   emit_int16(0x5D, (0xC0 | encode));
 7584 }
 7585 
 7586 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
 7587   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7588   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7589   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7590   emit_int16(0x5D, (0xC0 | encode));
 7591 }
 7592 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7593   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7594   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7595   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7596   emit_int16(0x5D, (0xC0 | encode));
 7597 }
 7598 
 7599 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
 7600   assert(VM_Version::supports_sse4_1(), "");
 7601   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7602   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7603   emit_int16(0x3C, (0xC0 | encode));
 7604 }
 7605 
 7606 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7607   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7608         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7609   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7610   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7611   emit_int16(0x3C, (0xC0 | encode));
 7612 }
 7613 
 7614 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
 7615   assert(VM_Version::supports_sse2(), "");
 7616   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7617   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7618   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7619 }
 7620 
 7621 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7622   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7623         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
 7624   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7625   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7626   emit_int16((unsigned char)0xEE, (0xC0 | encode));
 7627 }
 7628 
 7629 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
 7630   assert(VM_Version::supports_sse4_1(), "");
 7631   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7632   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7633   emit_int16(0x3D, (0xC0 | encode));
 7634 }
 7635 
 7636 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7637   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 7638         (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
 7639   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7640   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7641   emit_int16(0x3D, (0xC0 | encode));
 7642 }
 7643 
 7644 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7645   assert(UseAVX > 2, "requires AVX512F");
 7646   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7647   attributes.set_is_evex_instruction();
 7648   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7649   emit_int16(0x3D, (0xC0 | encode));
 7650 }
 7651 
 7652 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
 7653   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7656   emit_int16(0x5F, (0xC0 | encode));
 7657 }
 7658 
 7659 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7660   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7661   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7662   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 7663   emit_int16(0x5F, (0xC0 | encode));
 7664 }
 7665 
 7666 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
 7667   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 7668   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7669   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7670   emit_int16(0x5F, (0xC0 | encode));
 7671 }
 7672 
 7673 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7674   assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
 7675   InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7676   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7677   emit_int16(0x5F, (0xC0 | encode));
 7678 }
 7679 
 7680 // Shift packed integers left by specified number of bits.
 7681 void Assembler::psllw(XMMRegister dst, int shift) {
 7682   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7683   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7684   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7685   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7686   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7687 }
 7688 
 7689 void Assembler::pslld(XMMRegister dst, int shift) {
 7690   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7691   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7692   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7693   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7694   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7695 }
 7696 
 7697 void Assembler::psllq(XMMRegister dst, int shift) {
 7698   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7699   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7700   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7701   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7702   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7703 }
 7704 
 7705 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
 7706   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7707   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7708   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7709   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7710 }
 7711 
 7712 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
 7713   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7715   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7716   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7717 }
 7718 
 7719 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
 7720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7721   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7722   attributes.set_rex_vex_w_reverted();
 7723   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7724   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7725 }
 7726 
 7727 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7728   assert(UseAVX > 0, "requires some form of AVX");
 7729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7730   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
 7731   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7732   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7733 }
 7734 
 7735 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7736   assert(UseAVX > 0, "requires some form of AVX");
 7737   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7738   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7739   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
 7740   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7741   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7742 }
 7743 
 7744 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7745   assert(UseAVX > 0, "requires some form of AVX");
 7746   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7747   attributes.set_rex_vex_w_reverted();
 7748   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
 7749   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7750   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7751 }
 7752 
 7753 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7754   assert(UseAVX > 0, "requires some form of AVX");
 7755   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7756   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7757   emit_int16((unsigned char)0xF1, (0xC0 | encode));
 7758 }
 7759 
 7760 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7761   assert(UseAVX > 0, "requires some form of AVX");
 7762   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7763   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7764   emit_int16((unsigned char)0xF2, (0xC0 | encode));
 7765 }
 7766 
 7767 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7768   assert(UseAVX > 0, "requires some form of AVX");
 7769   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7770   attributes.set_rex_vex_w_reverted();
 7771   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7772   emit_int16((unsigned char)0xF3, (0xC0 | encode));
 7773 }
 7774 
 7775 // Shift packed integers logically right by specified number of bits.
 7776 void Assembler::psrlw(XMMRegister dst, int shift) {
 7777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7778   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7779   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7780   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7781   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7782 }
 7783 
 7784 void Assembler::psrld(XMMRegister dst, int shift) {
 7785   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7786   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7787   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7788   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7789   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7790 }
 7791 
 7792 void Assembler::psrlq(XMMRegister dst, int shift) {
 7793   // Do not confuse it with psrldq SSE2 instruction which
 7794   // shifts 128 bit value in xmm register by number of bytes.
 7795   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7796   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7797   attributes.set_rex_vex_w_reverted();
 7798   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7799   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7800   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7801 }
 7802 
 7803 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
 7804   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7805   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7806   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7807   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7808 }
 7809 
 7810 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
 7811   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7813   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7814   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7815 }
 7816 
 7817 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
 7818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7819   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7820   attributes.set_rex_vex_w_reverted();
 7821   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7822   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7823 }
 7824 
 7825 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7826   assert(UseAVX > 0, "requires some form of AVX");
 7827   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7828   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
 7829   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7830   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7831 }
 7832 
 7833 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7834   assert(UseAVX > 0, "requires some form of AVX");
 7835   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7836   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
 7837   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7838   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7839 }
 7840 
 7841 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7842   assert(UseAVX > 0, "requires some form of AVX");
 7843   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7844   attributes.set_rex_vex_w_reverted();
 7845   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
 7846   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7847   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 7848 }
 7849 
 7850 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7851   assert(UseAVX > 0, "requires some form of AVX");
 7852   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7853   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7854   emit_int16((unsigned char)0xD1, (0xC0 | encode));
 7855 }
 7856 
 7857 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7858   assert(UseAVX > 0, "requires some form of AVX");
 7859   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7860   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7861   emit_int16((unsigned char)0xD2, (0xC0 | encode));
 7862 }
 7863 
 7864 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7865   assert(UseAVX > 0, "requires some form of AVX");
 7866   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7867   attributes.set_rex_vex_w_reverted();
 7868   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7869   emit_int16((unsigned char)0xD3, (0xC0 | encode));
 7870 }
 7871 
 7872 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7873   assert(VM_Version::supports_avx512bw(), "");
 7874   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7875   attributes.set_is_evex_instruction();
 7876   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7877   emit_int16(0x10, (0xC0 | encode));
 7878 }
 7879 
 7880 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7881   assert(VM_Version::supports_avx512bw(), "");
 7882   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7883   attributes.set_is_evex_instruction();
 7884   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 7885   emit_int16(0x12, (0xC0 | encode));
 7886 }
 7887 
 7888 // Shift packed integers arithmetically right by specified number of bits.
 7889 void Assembler::psraw(XMMRegister dst, int shift) {
 7890   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7891   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7892   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7893   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7894   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7895 }
 7896 
 7897 void Assembler::psrad(XMMRegister dst, int shift) {
 7898   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7899   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7900   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
 7901   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7902   emit_int8(0x72);
 7903   emit_int8((0xC0 | encode));
 7904   emit_int8(shift & 0xFF);
 7905 }
 7906 
 7907 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
 7908   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7909   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7910   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7911   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7912 }
 7913 
 7914 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
 7915   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7916   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7917   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7918   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7919 }
 7920 
 7921 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7922   assert(UseAVX > 0, "requires some form of AVX");
 7923   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7924   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7925   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7926   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 7927 }
 7928 
 7929 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7930   assert(UseAVX > 0, "requires some form of AVX");
 7931   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7932   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
 7933   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7934   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 7935 }
 7936 
 7937 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7938   assert(UseAVX > 0, "requires some form of AVX");
 7939   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 7940   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7941   emit_int16((unsigned char)0xE1, (0xC0 | encode));
 7942 }
 7943 
 7944 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7945   assert(UseAVX > 0, "requires some form of AVX");
 7946   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7947   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7948   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7949 }
 7950 
 7951 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 7952   assert(UseAVX > 2, "requires AVX512");
 7953   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7954   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7955   attributes.set_is_evex_instruction();
 7956   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7957   emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
 7958 }
 7959 
 7960 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 7961   assert(UseAVX > 2, "requires AVX512");
 7962   assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
 7963   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7964   attributes.set_is_evex_instruction();
 7965   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7966   emit_int16((unsigned char)0xE2, (0xC0 | encode));
 7967 }
 7968 
 7969 // logical operations packed integers
 7970 void Assembler::pand(XMMRegister dst, XMMRegister src) {
 7971   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 7972   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7973   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7974   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7975 }
 7976 
 7977 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7978   assert(UseAVX > 0, "requires some form of AVX");
 7979   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7980   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7981   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 7982 }
 7983 
 7984 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7985   assert(UseAVX > 0, "requires some form of AVX");
 7986   InstructionMark im(this);
 7987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 7988   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 7989   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 7990   emit_int8((unsigned char)0xDB);
 7991   emit_operand(dst, src, 0);
 7992 }
 7993 
 7994 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 7995   evpandq(dst, k0, nds, src, false, vector_len);
 7996 }
 7997 
 7998 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 7999   evpandq(dst, k0, nds, src, false, vector_len);
 8000 }
 8001 
 8002 //Variable Shift packed integers logically left.
 8003 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8004   assert(UseAVX > 1, "requires AVX2");
 8005   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8006   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8007   emit_int16(0x47, (0xC0 | encode));
 8008 }
 8009 
 8010 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8011   assert(UseAVX > 1, "requires AVX2");
 8012   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8013   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8014   emit_int16(0x47, (0xC0 | encode));
 8015 }
 8016 
 8017 //Variable Shift packed integers logically right.
 8018 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8019   assert(UseAVX > 1, "requires AVX2");
 8020   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8021   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8022   emit_int16(0x45, (0xC0 | encode));
 8023 }
 8024 
 8025 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8026   assert(UseAVX > 1, "requires AVX2");
 8027   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8028   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8029   emit_int16(0x45, (0xC0 | encode));
 8030 }
 8031 
 8032 //Variable right Shift arithmetic packed integers .
 8033 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8034   assert(UseAVX > 1, "requires AVX2");
 8035   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8036   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8037   emit_int16(0x46, (0xC0 | encode));
 8038 }
 8039 
 8040 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8041   assert(VM_Version::supports_avx512bw(), "");
 8042   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8043   attributes.set_is_evex_instruction();
 8044   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8045   emit_int16(0x11, (0xC0 | encode));
 8046 }
 8047 
 8048 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8049   assert(UseAVX > 2, "requires AVX512");
 8050   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8051   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8052   attributes.set_is_evex_instruction();
 8053   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8054   emit_int16(0x46, (0xC0 | encode));
 8055 }
 8056 
 8057 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8058   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 8059   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8060   attributes.set_is_evex_instruction();
 8061   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8062   emit_int16(0x71, (0xC0 | encode));
 8063 }
 8064 
 8065 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8066   assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
 8067   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8068   attributes.set_is_evex_instruction();
 8069   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8070   emit_int16(0x73, (0xC0 | encode));
 8071 }
 8072 
 8073 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
 8074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 8075   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8076   attributes.set_rex_vex_w_reverted();
 8077   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8078   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 8079 }
 8080 
 8081 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8082   assert(UseAVX > 0, "requires some form of AVX");
 8083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8084   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8085   emit_int16((unsigned char)0xDF, (0xC0 | encode));
 8086 }
 8087 
 8088 void Assembler::por(XMMRegister dst, XMMRegister src) {
 8089   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 8090   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8091   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8092   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8093 }
 8094 
 8095 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8096   assert(UseAVX > 0, "requires some form of AVX");
 8097   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8098   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8099   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8100 }
 8101 
 8102 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8103   assert(UseAVX > 0, "requires some form of AVX");
 8104   InstructionMark im(this);
 8105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8106   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 8107   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8108   emit_int8((unsigned char)0xEB);
 8109   emit_operand(dst, src, 0);
 8110 }
 8111 
 8112 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8113   evporq(dst, k0, nds, src, false, vector_len);
 8114 }
 8115 
 8116 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8117   evporq(dst, k0, nds, src, false, vector_len);
 8118 }
 8119 
 8120 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8121   assert(VM_Version::supports_evex(), "");
 8122   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 8123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8124   attributes.set_is_evex_instruction();
 8125   attributes.set_embedded_opmask_register_specifier(mask);
 8126   if (merge) {
 8127     attributes.reset_is_clear_context();
 8128   }
 8129   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8130   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8131 }
 8132 
 8133 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8134   assert(VM_Version::supports_evex(), "");
 8135   // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
 8136   InstructionMark im(this);
 8137   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8138   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
 8139   attributes.set_is_evex_instruction();
 8140   attributes.set_embedded_opmask_register_specifier(mask);
 8141   if (merge) {
 8142     attributes.reset_is_clear_context();
 8143   }
 8144   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8145   emit_int8((unsigned char)0xEB);
 8146   emit_operand(dst, src, 0);
 8147 }
 8148 
 8149 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 8150   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 8151   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8152   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8153   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8154 }
 8155 
 8156 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8157   assert(UseAVX > 0, "requires some form of AVX");
 8158   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 8159          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 8160          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 8161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8162   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8163   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8164 }
 8165 
 8166 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8167   assert(UseAVX > 0, "requires some form of AVX");
 8168   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
 8169          vector_len == AVX_256bit ? VM_Version::supports_avx2() :
 8170          vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
 8171   InstructionMark im(this);
 8172   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8173   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
 8174   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8175   emit_int8((unsigned char)0xEF);
 8176   emit_operand(dst, src, 0);
 8177 }
 8178 
 8179 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8180   assert(UseAVX > 2, "requires some form of EVEX");
 8181   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8182   attributes.set_rex_vex_w_reverted();
 8183   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8184   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8185 }
 8186 
 8187 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8188   // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
 8189   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8190   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8191   attributes.set_is_evex_instruction();
 8192   attributes.set_embedded_opmask_register_specifier(mask);
 8193   if (merge) {
 8194     attributes.reset_is_clear_context();
 8195   }
 8196   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8197   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8198 }
 8199 
 8200 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8201   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8202   InstructionMark im(this);
 8203   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8204   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8205   attributes.set_is_evex_instruction();
 8206   attributes.set_embedded_opmask_register_specifier(mask);
 8207   if (merge) {
 8208     attributes.reset_is_clear_context();
 8209   }
 8210   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8211   emit_int8((unsigned char)0xEF);
 8212   emit_operand(dst, src, 0);
 8213 }
 8214 
 8215 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8216   // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
 8217   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8218   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8219   attributes.set_is_evex_instruction();
 8220   attributes.set_embedded_opmask_register_specifier(mask);
 8221   if (merge) {
 8222     attributes.reset_is_clear_context();
 8223   }
 8224   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8225   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8226 }
 8227 
 8228 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8229   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8230   InstructionMark im(this);
 8231   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8232   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8233   attributes.set_is_evex_instruction();
 8234   attributes.set_embedded_opmask_register_specifier(mask);
 8235   if (merge) {
 8236     attributes.reset_is_clear_context();
 8237   }
 8238   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8239   emit_int8((unsigned char)0xEF);
 8240   emit_operand(dst, src, 0);
 8241 }
 8242 
 8243 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8244   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8245   InstructionMark im(this);
 8246   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8247   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8248   attributes.set_is_evex_instruction();
 8249   attributes.set_embedded_opmask_register_specifier(mask);
 8250   if (merge) {
 8251     attributes.reset_is_clear_context();
 8252   }
 8253   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8254   emit_int8((unsigned char)0xDB);
 8255   emit_operand(dst, src, 0);
 8256 }
 8257 
 8258 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8259   assert(VM_Version::supports_evex(), "requires AVX512F");
 8260   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8261   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8262   attributes.set_is_evex_instruction();
 8263   attributes.set_embedded_opmask_register_specifier(mask);
 8264   if (merge) {
 8265     attributes.reset_is_clear_context();
 8266   }
 8267   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8268   emit_int16((unsigned char)0xDB, (0xC0 | encode));
 8269 }
 8270 
 8271 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8272   assert(VM_Version::supports_evex(), "requires AVX512F");
 8273   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8274   InstructionMark im(this);
 8275   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8276   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8277   attributes.set_is_evex_instruction();
 8278   attributes.set_embedded_opmask_register_specifier(mask);
 8279   if (merge) {
 8280     attributes.reset_is_clear_context();
 8281   }
 8282   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8283   emit_int8((unsigned char)0xDB);
 8284   emit_operand(dst, src, 0);
 8285 }
 8286 
 8287 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 8288   assert(VM_Version::supports_evex(), "requires AVX512F");
 8289   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8290   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 8291   attributes.set_is_evex_instruction();
 8292   attributes.set_embedded_opmask_register_specifier(mask);
 8293   if (merge) {
 8294     attributes.reset_is_clear_context();
 8295   }
 8296   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8297   emit_int16((unsigned char)0xEB, (0xC0 | encode));
 8298 }
 8299 
 8300 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 8301   assert(VM_Version::supports_evex(), "requires AVX512F");
 8302   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8303   InstructionMark im(this);
 8304   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8305   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 8306   attributes.set_is_evex_instruction();
 8307   attributes.set_embedded_opmask_register_specifier(mask);
 8308   if (merge) {
 8309     attributes.reset_is_clear_context();
 8310   }
 8311   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8312   emit_int8((unsigned char)0xEB);
 8313   emit_operand(dst, src, 0);
 8314 }
 8315 
 8316 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8317   assert(VM_Version::supports_evex(), "requires EVEX support");
 8318   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8319   attributes.set_is_evex_instruction();
 8320   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8321   emit_int16((unsigned char)0xEF, (0xC0 | encode));
 8322 }
 8323 
 8324 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 8325   assert(VM_Version::supports_evex(), "requires EVEX support");
 8326   assert(dst != xnoreg, "sanity");
 8327   InstructionMark im(this);
 8328   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8329   attributes.set_is_evex_instruction();
 8330   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8331   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8332   emit_int8((unsigned char)0xEF);
 8333   emit_operand(dst, src, 0);
 8334 }
 8335 
 8336 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8337   assert(VM_Version::supports_evex(), "requires EVEX support");
 8338   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8339   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8340   attributes.set_is_evex_instruction();
 8341   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8342   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8343 }
 8344 
 8345 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8346   assert(VM_Version::supports_evex(), "requires EVEX support");
 8347   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8348   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8349   attributes.set_is_evex_instruction();
 8350   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8351   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8352 }
 8353 
 8354 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8355   assert(VM_Version::supports_evex(), "requires EVEX support");
 8356   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8357   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8358   attributes.set_is_evex_instruction();
 8359   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8360   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8361 }
 8362 
 8363 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
 8364   assert(VM_Version::supports_evex(), "requires EVEX support");
 8365   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8366   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8367   attributes.set_is_evex_instruction();
 8368   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8369   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 8370 }
 8371 
 8372 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8373   assert(VM_Version::supports_evex(), "requires EVEX support");
 8374   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8375   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8376   attributes.set_is_evex_instruction();
 8377   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8378   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8379 }
 8380 
 8381 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8382   assert(VM_Version::supports_evex(), "requires EVEX support");
 8383   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8384   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8385   attributes.set_is_evex_instruction();
 8386   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8387   emit_int16(0x15, (unsigned char)(0xC0 | encode));
 8388 }
 8389 
 8390 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8391   assert(VM_Version::supports_evex(), "requires EVEX support");
 8392   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8393   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8394   attributes.set_is_evex_instruction();
 8395   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8396   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8397 }
 8398 
 8399 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
 8400   assert(VM_Version::supports_evex(), "requires EVEX support");
 8401   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8402   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8403   attributes.set_is_evex_instruction();
 8404   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8405   emit_int16(0x14, (unsigned char)(0xC0 | encode));
 8406 }
 8407 
 8408 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8409   assert(VM_Version::supports_avx512cd(), "");
 8410   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8411   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8412   attributes.set_is_evex_instruction();
 8413   attributes.set_embedded_opmask_register_specifier(mask);
 8414   if (merge) {
 8415     attributes.reset_is_clear_context();
 8416   }
 8417   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8418   emit_int16(0x44, (0xC0 | encode));
 8419 }
 8420 
 8421 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8422   assert(VM_Version::supports_avx512cd(), "");
 8423   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8424   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8425   attributes.set_is_evex_instruction();
 8426   attributes.set_embedded_opmask_register_specifier(mask);
 8427   if (merge) {
 8428     attributes.reset_is_clear_context();
 8429   }
 8430   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8431   emit_int16(0x44, (0xC0 | encode));
 8432 }
 8433 
 8434 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8435   assert(VM_Version::supports_evex(), "requires EVEX support");
 8436   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8437   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8438   attributes.set_is_evex_instruction();
 8439   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8440   emit_int8(0x25);
 8441   emit_int8((unsigned char)(0xC0 | encode));
 8442   emit_int8(imm8);
 8443 }
 8444 
 8445 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8446   assert(VM_Version::supports_evex(), "requires EVEX support");
 8447   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8448   assert(dst != xnoreg, "sanity");
 8449   InstructionMark im(this);
 8450   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8451   attributes.set_is_evex_instruction();
 8452   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8453   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8454   emit_int8(0x25);
 8455   emit_operand(dst, src3, 1);
 8456   emit_int8(imm8);
 8457 }
 8458 
 8459 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
 8460   assert(VM_Version::supports_evex(), "requires AVX512F");
 8461   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
 8462   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8463   attributes.set_is_evex_instruction();
 8464   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8465   emit_int8(0x25);
 8466   emit_int8((unsigned char)(0xC0 | encode));
 8467   emit_int8(imm8);
 8468 }
 8469 
 8470 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
 8471   assert(VM_Version::supports_evex(), "requires EVEX support");
 8472   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
 8473   assert(dst != xnoreg, "sanity");
 8474   InstructionMark im(this);
 8475   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8476   attributes.set_is_evex_instruction();
 8477   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
 8478   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8479   emit_int8(0x25);
 8480   emit_operand(dst, src3, 1);
 8481   emit_int8(imm8);
 8482 }
 8483 
 8484 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8485   assert(VM_Version::supports_evex(), "");
 8486   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8487   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8488   attributes.set_is_evex_instruction();
 8489   attributes.set_embedded_opmask_register_specifier(mask);
 8490   if (merge) {
 8491     attributes.reset_is_clear_context();
 8492   }
 8493   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8494   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8495 }
 8496 
 8497 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8498   assert(VM_Version::supports_evex(), "");
 8499   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8500   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8501   attributes.set_is_evex_instruction();
 8502   attributes.set_embedded_opmask_register_specifier(mask);
 8503   if (merge) {
 8504     attributes.reset_is_clear_context();
 8505   }
 8506   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8507   emit_int16((unsigned char)0x88, (0xC0 | encode));
 8508 }
 8509 
 8510 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8511   assert(VM_Version::supports_avx512_vbmi2(), "");
 8512   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8514   attributes.set_is_evex_instruction();
 8515   attributes.set_embedded_opmask_register_specifier(mask);
 8516   if (merge) {
 8517     attributes.reset_is_clear_context();
 8518   }
 8519   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8520   emit_int16(0x62, (0xC0 | encode));
 8521 }
 8522 
 8523 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8524   assert(VM_Version::supports_avx512_vbmi2(), "");
 8525   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8526   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8527   attributes.set_is_evex_instruction();
 8528   attributes.set_embedded_opmask_register_specifier(mask);
 8529   if (merge) {
 8530     attributes.reset_is_clear_context();
 8531   }
 8532   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8533   emit_int16(0x62, (0xC0 | encode));
 8534 }
 8535 
 8536 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8537   assert(VM_Version::supports_evex(), "");
 8538   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8539   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8540   attributes.set_is_evex_instruction();
 8541   attributes.set_embedded_opmask_register_specifier(mask);
 8542   if (merge) {
 8543     attributes.reset_is_clear_context();
 8544   }
 8545   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8546   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8547 }
 8548 
 8549 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 8550   assert(VM_Version::supports_evex(), "");
 8551   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 8552   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 8553   attributes.set_is_evex_instruction();
 8554   attributes.set_embedded_opmask_register_specifier(mask);
 8555   if (merge) {
 8556     attributes.reset_is_clear_context();
 8557   }
 8558   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8559   emit_int16((unsigned char)0x89, (0xC0 | encode));
 8560 }
 8561 
 8562 // vinserti forms
 8563 
 8564 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8565   assert(VM_Version::supports_avx2(), "");
 8566   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8567   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8568   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8569   // last byte:
 8570   // 0x00 - insert into lower 128 bits
 8571   // 0x01 - insert into upper 128 bits
 8572   emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
 8573 }
 8574 
 8575 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8576   assert(VM_Version::supports_avx2(), "");
 8577   assert(dst != xnoreg, "sanity");
 8578   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8579   InstructionMark im(this);
 8580   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8581   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8582   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8583   emit_int8(0x38);
 8584   emit_operand(dst, src, 1);
 8585   // 0x00 - insert into lower 128 bits
 8586   // 0x01 - insert into upper 128 bits
 8587   emit_int8(imm8 & 0x01);
 8588 }
 8589 
 8590 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8591   assert(VM_Version::supports_evex(), "");
 8592   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8593   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8594   attributes.set_is_evex_instruction();
 8595   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8596   // imm8:
 8597   // 0x00 - insert into q0 128 bits (0..127)
 8598   // 0x01 - insert into q1 128 bits (128..255)
 8599   // 0x02 - insert into q2 128 bits (256..383)
 8600   // 0x03 - insert into q3 128 bits (384..511)
 8601   emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
 8602 }
 8603 
 8604 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8605   assert(VM_Version::supports_evex(), "");
 8606   assert(dst != xnoreg, "sanity");
 8607   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8608   InstructionMark im(this);
 8609   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8610   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8611   attributes.set_is_evex_instruction();
 8612   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8613   emit_int8(0x18);
 8614   emit_operand(dst, src, 1);
 8615   // 0x00 - insert into q0 128 bits (0..127)
 8616   // 0x01 - insert into q1 128 bits (128..255)
 8617   // 0x02 - insert into q2 128 bits (256..383)
 8618   // 0x03 - insert into q3 128 bits (384..511)
 8619   emit_int8(imm8 & 0x03);
 8620 }
 8621 
 8622 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8623   assert(VM_Version::supports_evex(), "");
 8624   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8625   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8626   attributes.set_is_evex_instruction();
 8627   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8628   //imm8:
 8629   // 0x00 - insert into lower 256 bits
 8630   // 0x01 - insert into upper 256 bits
 8631   emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
 8632 }
 8633 
 8634 
 8635 // vinsertf forms
 8636 
 8637 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8638   assert(VM_Version::supports_avx(), "");
 8639   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8640   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8641   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8642   // imm8:
 8643   // 0x00 - insert into lower 128 bits
 8644   // 0x01 - insert into upper 128 bits
 8645   emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
 8646 }
 8647 
 8648 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8649   assert(VM_Version::supports_avx(), "");
 8650   assert(dst != xnoreg, "sanity");
 8651   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8652   InstructionMark im(this);
 8653   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8654   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8655   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8656   emit_int8(0x18);
 8657   emit_operand(dst, src, 1);
 8658   // 0x00 - insert into lower 128 bits
 8659   // 0x01 - insert into upper 128 bits
 8660   emit_int8(imm8 & 0x01);
 8661 }
 8662 
 8663 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8664   assert(VM_Version::supports_evex(), "");
 8665   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8666   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8667   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8668   // imm8:
 8669   // 0x00 - insert into q0 128 bits (0..127)
 8670   // 0x01 - insert into q1 128 bits (128..255)
 8671   // 0x02 - insert into q0 128 bits (256..383)
 8672   // 0x03 - insert into q1 128 bits (384..512)
 8673   emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
 8674 }
 8675 
 8676 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8677   assert(VM_Version::supports_evex(), "");
 8678   assert(dst != xnoreg, "sanity");
 8679   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8680   InstructionMark im(this);
 8681   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8682   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8683   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8684   emit_int8(0x18);
 8685   emit_operand(dst, src, 1);
 8686   // 0x00 - insert into q0 128 bits (0..127)
 8687   // 0x01 - insert into q1 128 bits (128..255)
 8688   // 0x02 - insert into q0 128 bits (256..383)
 8689   // 0x03 - insert into q1 128 bits (384..512)
 8690   emit_int8(imm8 & 0x03);
 8691 }
 8692 
 8693 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
 8694   assert(VM_Version::supports_evex(), "");
 8695   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8696   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8697   attributes.set_is_evex_instruction();
 8698   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8699   // imm8:
 8700   // 0x00 - insert into lower 256 bits
 8701   // 0x01 - insert into upper 256 bits
 8702   emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
 8703 }
 8704 
 8705 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
 8706   assert(VM_Version::supports_evex(), "");
 8707   assert(dst != xnoreg, "sanity");
 8708   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8709   InstructionMark im(this);
 8710   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8711   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8712   attributes.set_is_evex_instruction();
 8713   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8714   emit_int8(0x1A);
 8715   emit_operand(dst, src, 1);
 8716   // 0x00 - insert into lower 256 bits
 8717   // 0x01 - insert into upper 256 bits
 8718   emit_int8(imm8 & 0x01);
 8719 }
 8720 
 8721 
 8722 // vextracti forms
 8723 
 8724 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8725   assert(VM_Version::supports_avx2(), "");
 8726   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8727   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8728   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8729   // imm8:
 8730   // 0x00 - extract from lower 128 bits
 8731   // 0x01 - extract from upper 128 bits
 8732   emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
 8733 }
 8734 
 8735 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
 8736   assert(VM_Version::supports_avx2(), "");
 8737   assert(src != xnoreg, "sanity");
 8738   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8739   InstructionMark im(this);
 8740   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8741   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8742   attributes.reset_is_clear_context();
 8743   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8744   emit_int8(0x39);
 8745   emit_operand(src, dst, 1);
 8746   // 0x00 - extract from lower 128 bits
 8747   // 0x01 - extract from upper 128 bits
 8748   emit_int8(imm8 & 0x01);
 8749 }
 8750 
 8751 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8752   assert(VM_Version::supports_evex(), "");
 8753   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8754   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8755   attributes.set_is_evex_instruction();
 8756   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8757   // imm8:
 8758   // 0x00 - extract from bits 127:0
 8759   // 0x01 - extract from bits 255:128
 8760   // 0x02 - extract from bits 383:256
 8761   // 0x03 - extract from bits 511:384
 8762   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8763 }
 8764 
 8765 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8766   assert(VM_Version::supports_evex(), "");
 8767   assert(src != xnoreg, "sanity");
 8768   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8769   InstructionMark im(this);
 8770   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8771   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8772   attributes.reset_is_clear_context();
 8773   attributes.set_is_evex_instruction();
 8774   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8775   emit_int8(0x39);
 8776   emit_operand(src, dst, 1);
 8777   // 0x00 - extract from bits 127:0
 8778   // 0x01 - extract from bits 255:128
 8779   // 0x02 - extract from bits 383:256
 8780   // 0x03 - extract from bits 511:384
 8781   emit_int8(imm8 & 0x03);
 8782 }
 8783 
 8784 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8785   assert(VM_Version::supports_avx512dq(), "");
 8786   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8787   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8788   attributes.set_is_evex_instruction();
 8789   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8790   // imm8:
 8791   // 0x00 - extract from bits 127:0
 8792   // 0x01 - extract from bits 255:128
 8793   // 0x02 - extract from bits 383:256
 8794   // 0x03 - extract from bits 511:384
 8795   emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
 8796 }
 8797 
 8798 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8799   assert(VM_Version::supports_evex(), "");
 8800   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8801   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8802   attributes.set_is_evex_instruction();
 8803   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8804   // imm8:
 8805   // 0x00 - extract from lower 256 bits
 8806   // 0x01 - extract from upper 256 bits
 8807   emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
 8808 }
 8809 
 8810 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8811   assert(VM_Version::supports_evex(), "");
 8812   assert(src != xnoreg, "sanity");
 8813   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8814   InstructionMark im(this);
 8815   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8816   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
 8817   attributes.reset_is_clear_context();
 8818   attributes.set_is_evex_instruction();
 8819   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8820   emit_int8(0x38);
 8821   emit_operand(src, dst, 1);
 8822   // 0x00 - extract from lower 256 bits
 8823   // 0x01 - extract from upper 256 bits
 8824   emit_int8(imm8 & 0x01);
 8825 }
 8826 // vextractf forms
 8827 
 8828 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8829   assert(VM_Version::supports_avx(), "");
 8830   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8831   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8832   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8833   // imm8:
 8834   // 0x00 - extract from lower 128 bits
 8835   // 0x01 - extract from upper 128 bits
 8836   emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
 8837 }
 8838 
 8839 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
 8840   assert(VM_Version::supports_avx(), "");
 8841   assert(src != xnoreg, "sanity");
 8842   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8843   InstructionMark im(this);
 8844   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8845   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8846   attributes.reset_is_clear_context();
 8847   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8848   emit_int8(0x19);
 8849   emit_operand(src, dst, 1);
 8850   // 0x00 - extract from lower 128 bits
 8851   // 0x01 - extract from upper 128 bits
 8852   emit_int8(imm8 & 0x01);
 8853 }
 8854 
 8855 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8856   assert(VM_Version::supports_evex(), "");
 8857   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8858   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8859   attributes.set_is_evex_instruction();
 8860   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8861   // imm8:
 8862   // 0x00 - extract from bits 127:0
 8863   // 0x01 - extract from bits 255:128
 8864   // 0x02 - extract from bits 383:256
 8865   // 0x03 - extract from bits 511:384
 8866   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8867 }
 8868 
 8869 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
 8870   assert(VM_Version::supports_evex(), "");
 8871   assert(src != xnoreg, "sanity");
 8872   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8873   InstructionMark im(this);
 8874   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8875   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
 8876   attributes.reset_is_clear_context();
 8877   attributes.set_is_evex_instruction();
 8878   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8879   emit_int8(0x19);
 8880   emit_operand(src, dst, 1);
 8881   // 0x00 - extract from bits 127:0
 8882   // 0x01 - extract from bits 255:128
 8883   // 0x02 - extract from bits 383:256
 8884   // 0x03 - extract from bits 511:384
 8885   emit_int8(imm8 & 0x03);
 8886 }
 8887 
 8888 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8889   assert(VM_Version::supports_avx512dq(), "");
 8890   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8891   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8892   attributes.set_is_evex_instruction();
 8893   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8894   // imm8:
 8895   // 0x00 - extract from bits 127:0
 8896   // 0x01 - extract from bits 255:128
 8897   // 0x02 - extract from bits 383:256
 8898   // 0x03 - extract from bits 511:384
 8899   emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
 8900 }
 8901 
 8902 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
 8903   assert(VM_Version::supports_evex(), "");
 8904   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8905   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8906   attributes.set_is_evex_instruction();
 8907   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8908   // imm8:
 8909   // 0x00 - extract from lower 256 bits
 8910   // 0x01 - extract from upper 256 bits
 8911   emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
 8912 }
 8913 
 8914 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
 8915   assert(VM_Version::supports_evex(), "");
 8916   assert(src != xnoreg, "sanity");
 8917   assert(imm8 <= 0x01, "imm8: %u", imm8);
 8918   InstructionMark im(this);
 8919   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8920   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
 8921   attributes.reset_is_clear_context();
 8922   attributes.set_is_evex_instruction();
 8923   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8924   emit_int8(0x1B);
 8925   emit_operand(src, dst, 1);
 8926   // 0x00 - extract from lower 256 bits
 8927   // 0x01 - extract from upper 256 bits
 8928   emit_int8(imm8 & 0x01);
 8929 }
 8930 
 8931 void Assembler::extractps(Register dst, XMMRegister src, uint8_t imm8) {
 8932   assert(VM_Version::supports_sse4_1(), "");
 8933   assert(imm8 <= 0x03, "imm8: %u", imm8);
 8934   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 8935   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
 8936   // imm8:
 8937   // 0x00 - extract from bits 31:0
 8938   // 0x01 - extract from bits 63:32
 8939   // 0x02 - extract from bits 95:64
 8940   // 0x03 - extract from bits 127:96
 8941   emit_int24(0x17, (0xC0 | encode), imm8 & 0x03);
 8942 }
 8943 
 8944 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8945 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
 8946   assert(VM_Version::supports_avx2(), "");
 8947   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8948   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8949   emit_int16(0x78, (0xC0 | encode));
 8950 }
 8951 
 8952 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
 8953   assert(VM_Version::supports_avx2(), "");
 8954   assert(dst != xnoreg, "sanity");
 8955   InstructionMark im(this);
 8956   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8957   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
 8958   // swap src<->dst for encoding
 8959   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8960   emit_int8(0x78);
 8961   emit_operand(dst, src, 0);
 8962 }
 8963 
 8964 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
 8965 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
 8966   assert(VM_Version::supports_avx2(), "");
 8967   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8968   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8969   emit_int16(0x79, (0xC0 | encode));
 8970 }
 8971 
 8972 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
 8973   assert(VM_Version::supports_avx2(), "");
 8974   assert(dst != xnoreg, "sanity");
 8975   InstructionMark im(this);
 8976   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8977   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
 8978   // swap src<->dst for encoding
 8979   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 8980   emit_int8(0x79);
 8981   emit_operand(dst, src, 0);
 8982 }
 8983 
 8984 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8985   assert(UseAVX > 0, "requires some form of AVX");
 8986   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
 8987   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8988   emit_int16((unsigned char)0xF6, (0xC0 | encode));
 8989 }
 8990 
 8991 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8992   assert(UseAVX > 0, "requires some form of AVX");
 8993   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 8994   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 8995   emit_int16(0x69, (0xC0 | encode));
 8996 }
 8997 
 8998 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 8999   assert(UseAVX > 0, "requires some form of AVX");
 9000   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9001   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9002   emit_int16(0x61, (0xC0 | encode));
 9003 }
 9004 
 9005 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9006   assert(UseAVX > 0, "requires some form of AVX");
 9007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9008   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9009   emit_int16(0x6A, (0xC0 | encode));
 9010 }
 9011 
 9012 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 9013   assert(UseAVX > 0, "requires some form of AVX");
 9014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
 9015   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9016   emit_int16(0x62, (0xC0 | encode));
 9017 }
 9018 
 9019 // xmm/mem sourced byte/word/dword/qword replicate
 9020 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9021   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9023   attributes.set_is_evex_instruction();
 9024   attributes.set_embedded_opmask_register_specifier(mask);
 9025   if (merge) {
 9026     attributes.reset_is_clear_context();
 9027   }
 9028   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9029   emit_int16((unsigned char)0xFC, (0xC0 | encode));
 9030 }
 9031 
 9032 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9033   InstructionMark im(this);
 9034   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9035   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9036   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9037   attributes.set_is_evex_instruction();
 9038   attributes.set_embedded_opmask_register_specifier(mask);
 9039   if (merge) {
 9040     attributes.reset_is_clear_context();
 9041   }
 9042   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9043   emit_int8((unsigned char)0xFC);
 9044   emit_operand(dst, src, 0);
 9045 }
 9046 
 9047 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9048   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9049   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9050   attributes.set_is_evex_instruction();
 9051   attributes.set_embedded_opmask_register_specifier(mask);
 9052   if (merge) {
 9053     attributes.reset_is_clear_context();
 9054   }
 9055   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9056   emit_int16((unsigned char)0xFD, (0xC0 | encode));
 9057 }
 9058 
 9059 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9060   InstructionMark im(this);
 9061   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9062   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9063   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9064   attributes.set_is_evex_instruction();
 9065   attributes.set_embedded_opmask_register_specifier(mask);
 9066   if (merge) {
 9067     attributes.reset_is_clear_context();
 9068   }
 9069   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9070   emit_int8((unsigned char)0xFD);
 9071   emit_operand(dst, src, 0);
 9072 }
 9073 
 9074 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9075   assert(VM_Version::supports_evex(), "");
 9076   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9077   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9078   attributes.set_is_evex_instruction();
 9079   attributes.set_embedded_opmask_register_specifier(mask);
 9080   if (merge) {
 9081     attributes.reset_is_clear_context();
 9082   }
 9083   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9084   emit_int16((unsigned char)0xFE, (0xC0 | encode));
 9085 }
 9086 
 9087 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9088   InstructionMark im(this);
 9089   assert(VM_Version::supports_evex(), "");
 9090   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9091   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9092   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9093   attributes.set_is_evex_instruction();
 9094   attributes.set_embedded_opmask_register_specifier(mask);
 9095   if (merge) {
 9096     attributes.reset_is_clear_context();
 9097   }
 9098   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9099   emit_int8((unsigned char)0xFE);
 9100   emit_operand(dst, src, 0);
 9101 }
 9102 
 9103 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9104   assert(VM_Version::supports_evex(), "");
 9105   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9106   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9107   attributes.set_is_evex_instruction();
 9108   attributes.set_embedded_opmask_register_specifier(mask);
 9109   if (merge) {
 9110     attributes.reset_is_clear_context();
 9111   }
 9112   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9113   emit_int16((unsigned char)0xD4, (0xC0 | encode));
 9114 }
 9115 
 9116 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9117   InstructionMark im(this);
 9118   assert(VM_Version::supports_evex(), "");
 9119   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9120   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9121   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9122   attributes.set_is_evex_instruction();
 9123   attributes.set_embedded_opmask_register_specifier(mask);
 9124   if (merge) {
 9125     attributes.reset_is_clear_context();
 9126   }
 9127   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9128   emit_int8((unsigned char)0xD4);
 9129   emit_operand(dst, src, 0);
 9130 }
 9131 
 9132 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9133   assert(VM_Version::supports_evex(), "");
 9134   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9135   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9136   attributes.set_is_evex_instruction();
 9137   attributes.set_embedded_opmask_register_specifier(mask);
 9138   if (merge) {
 9139     attributes.reset_is_clear_context();
 9140   }
 9141   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9142   emit_int16(0x58, (0xC0 | encode));
 9143 }
 9144 
 9145 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9146   InstructionMark im(this);
 9147   assert(VM_Version::supports_evex(), "");
 9148   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9149   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9150   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9151   attributes.set_is_evex_instruction();
 9152   attributes.set_embedded_opmask_register_specifier(mask);
 9153   if (merge) {
 9154     attributes.reset_is_clear_context();
 9155   }
 9156   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9157   emit_int8(0x58);
 9158   emit_operand(dst, src, 0);
 9159 }
 9160 
 9161 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9162   assert(VM_Version::supports_evex(), "");
 9163   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9164   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9165   attributes.set_is_evex_instruction();
 9166   attributes.set_embedded_opmask_register_specifier(mask);
 9167   if (merge) {
 9168     attributes.reset_is_clear_context();
 9169   }
 9170   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9171   emit_int16(0x58, (0xC0 | encode));
 9172 }
 9173 
 9174 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9175   InstructionMark im(this);
 9176   assert(VM_Version::supports_evex(), "");
 9177   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9178   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9179   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9180   attributes.set_is_evex_instruction();
 9181   attributes.set_embedded_opmask_register_specifier(mask);
 9182   if (merge) {
 9183     attributes.reset_is_clear_context();
 9184   }
 9185   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9186   emit_int8(0x58);
 9187   emit_operand(dst, src, 0);
 9188 }
 9189 
 9190 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9191   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9192   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9193   attributes.set_is_evex_instruction();
 9194   attributes.set_embedded_opmask_register_specifier(mask);
 9195   if (merge) {
 9196     attributes.reset_is_clear_context();
 9197   }
 9198   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9199   emit_int16((unsigned char)0xF8, (0xC0 | encode));
 9200 }
 9201 
 9202 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9203   InstructionMark im(this);
 9204   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9205   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9206   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9207   attributes.set_is_evex_instruction();
 9208   attributes.set_embedded_opmask_register_specifier(mask);
 9209   if (merge) {
 9210     attributes.reset_is_clear_context();
 9211   }
 9212   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9213   emit_int8((unsigned char)0xF8);
 9214   emit_operand(dst, src, 0);
 9215 }
 9216 
 9217 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9218   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9219   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9220   attributes.set_is_evex_instruction();
 9221   attributes.set_embedded_opmask_register_specifier(mask);
 9222   if (merge) {
 9223     attributes.reset_is_clear_context();
 9224   }
 9225   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9226   emit_int16((unsigned char)0xF9, (0xC0 | encode));
 9227 }
 9228 
 9229 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9230   InstructionMark im(this);
 9231   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9232   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9233   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9234   attributes.set_is_evex_instruction();
 9235   attributes.set_embedded_opmask_register_specifier(mask);
 9236   if (merge) {
 9237     attributes.reset_is_clear_context();
 9238   }
 9239   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9240   emit_int8((unsigned char)0xF9);
 9241   emit_operand(dst, src, 0);
 9242 }
 9243 
 9244 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9245   assert(VM_Version::supports_evex(), "");
 9246   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9247   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9248   attributes.set_is_evex_instruction();
 9249   attributes.set_embedded_opmask_register_specifier(mask);
 9250   if (merge) {
 9251     attributes.reset_is_clear_context();
 9252   }
 9253   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9254   emit_int16((unsigned char)0xFA, (0xC0 | encode));
 9255 }
 9256 
 9257 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9258   InstructionMark im(this);
 9259   assert(VM_Version::supports_evex(), "");
 9260   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9261   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9262   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9263   attributes.set_is_evex_instruction();
 9264   attributes.set_embedded_opmask_register_specifier(mask);
 9265   if (merge) {
 9266     attributes.reset_is_clear_context();
 9267   }
 9268   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9269   emit_int8((unsigned char)0xFA);
 9270   emit_operand(dst, src, 0);
 9271 }
 9272 
 9273 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9274   assert(VM_Version::supports_evex(), "");
 9275   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9276   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9277   attributes.set_is_evex_instruction();
 9278   attributes.set_embedded_opmask_register_specifier(mask);
 9279   if (merge) {
 9280     attributes.reset_is_clear_context();
 9281   }
 9282   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9283   emit_int16((unsigned char)0xFB, (0xC0 | encode));
 9284 }
 9285 
 9286 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9287   InstructionMark im(this);
 9288   assert(VM_Version::supports_evex(), "");
 9289   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9290   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9291   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9292   attributes.set_is_evex_instruction();
 9293   attributes.set_embedded_opmask_register_specifier(mask);
 9294   if (merge) {
 9295     attributes.reset_is_clear_context();
 9296   }
 9297   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9298   emit_int8((unsigned char)0xFB);
 9299   emit_operand(dst, src, 0);
 9300 }
 9301 
 9302 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9303   assert(VM_Version::supports_evex(), "");
 9304   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9305   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9306   attributes.set_is_evex_instruction();
 9307   attributes.set_embedded_opmask_register_specifier(mask);
 9308   if (merge) {
 9309     attributes.reset_is_clear_context();
 9310   }
 9311   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9312   emit_int16(0x5C, (0xC0 | encode));
 9313 }
 9314 
 9315 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9316   InstructionMark im(this);
 9317   assert(VM_Version::supports_evex(), "");
 9318   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9319   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9320   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9321   attributes.set_is_evex_instruction();
 9322   attributes.set_embedded_opmask_register_specifier(mask);
 9323   if (merge) {
 9324     attributes.reset_is_clear_context();
 9325   }
 9326   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9327   emit_int8(0x5C);
 9328   emit_operand(dst, src, 0);
 9329 }
 9330 
 9331 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9332   assert(VM_Version::supports_evex(), "");
 9333   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9334   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9335   attributes.set_is_evex_instruction();
 9336   attributes.set_embedded_opmask_register_specifier(mask);
 9337   if (merge) {
 9338     attributes.reset_is_clear_context();
 9339   }
 9340   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9341   emit_int16(0x5C, (0xC0 | encode));
 9342 }
 9343 
 9344 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9345   InstructionMark im(this);
 9346   assert(VM_Version::supports_evex(), "");
 9347   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9348   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9349   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9350   attributes.set_is_evex_instruction();
 9351   attributes.set_embedded_opmask_register_specifier(mask);
 9352   if (merge) {
 9353     attributes.reset_is_clear_context();
 9354   }
 9355   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9356   emit_int8(0x5C);
 9357   emit_operand(dst, src, 0);
 9358 }
 9359 
 9360 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9361   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9362   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9363   attributes.set_is_evex_instruction();
 9364   attributes.set_embedded_opmask_register_specifier(mask);
 9365   if (merge) {
 9366     attributes.reset_is_clear_context();
 9367   }
 9368   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9369   emit_int16((unsigned char)0xD5, (0xC0 | encode));
 9370 }
 9371 
 9372 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9373   InstructionMark im(this);
 9374   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9375   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9376   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9377   attributes.set_is_evex_instruction();
 9378   attributes.set_embedded_opmask_register_specifier(mask);
 9379   if (merge) {
 9380     attributes.reset_is_clear_context();
 9381   }
 9382   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9383   emit_int8((unsigned char)0xD5);
 9384   emit_operand(dst, src, 0);
 9385 }
 9386 
 9387 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9388   assert(VM_Version::supports_evex(), "");
 9389   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9390   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9391   attributes.set_is_evex_instruction();
 9392   attributes.set_embedded_opmask_register_specifier(mask);
 9393   if (merge) {
 9394     attributes.reset_is_clear_context();
 9395   }
 9396   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9397   emit_int16(0x40, (0xC0 | encode));
 9398 }
 9399 
 9400 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9401   InstructionMark im(this);
 9402   assert(VM_Version::supports_evex(), "");
 9403   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9404   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9405   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9406   attributes.set_is_evex_instruction();
 9407   attributes.set_embedded_opmask_register_specifier(mask);
 9408   if (merge) {
 9409     attributes.reset_is_clear_context();
 9410   }
 9411   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9412   emit_int8(0x40);
 9413   emit_operand(dst, src, 0);
 9414 }
 9415 
 9416 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9417   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9418   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9419   attributes.set_is_evex_instruction();
 9420   attributes.set_embedded_opmask_register_specifier(mask);
 9421   if (merge) {
 9422     attributes.reset_is_clear_context();
 9423   }
 9424   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9425   emit_int16(0x40, (0xC0 | encode));
 9426 }
 9427 
 9428 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9429   InstructionMark im(this);
 9430   assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9431   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9432   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9433   attributes.set_is_evex_instruction();
 9434   attributes.set_embedded_opmask_register_specifier(mask);
 9435   if (merge) {
 9436     attributes.reset_is_clear_context();
 9437   }
 9438   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9439   emit_int8(0x40);
 9440   emit_operand(dst, src, 0);
 9441 }
 9442 
 9443 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9444   assert(VM_Version::supports_evex(), "");
 9445   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9446   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9447   attributes.set_is_evex_instruction();
 9448   attributes.set_embedded_opmask_register_specifier(mask);
 9449   if (merge) {
 9450     attributes.reset_is_clear_context();
 9451   }
 9452   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9453   emit_int16(0x59, (0xC0 | encode));
 9454 }
 9455 
 9456 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9457   InstructionMark im(this);
 9458   assert(VM_Version::supports_evex(), "");
 9459   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9460   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9461   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9462   attributes.set_is_evex_instruction();
 9463   attributes.set_embedded_opmask_register_specifier(mask);
 9464   if (merge) {
 9465     attributes.reset_is_clear_context();
 9466   }
 9467   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9468   emit_int8(0x59);
 9469   emit_operand(dst, src, 0);
 9470 }
 9471 
 9472 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9473   assert(VM_Version::supports_evex(), "");
 9474   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9475   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9476   attributes.set_is_evex_instruction();
 9477   attributes.set_embedded_opmask_register_specifier(mask);
 9478   if (merge) {
 9479     attributes.reset_is_clear_context();
 9480   }
 9481   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9482   emit_int16(0x59, (0xC0 | encode));
 9483 }
 9484 
 9485 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9486   InstructionMark im(this);
 9487   assert(VM_Version::supports_evex(), "");
 9488   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9489   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9490   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9491   attributes.set_is_evex_instruction();
 9492   attributes.set_embedded_opmask_register_specifier(mask);
 9493   if (merge) {
 9494     attributes.reset_is_clear_context();
 9495   }
 9496   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9497   emit_int8(0x59);
 9498   emit_operand(dst, src, 0);
 9499 }
 9500 
 9501 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9502   assert(VM_Version::supports_evex(), "");
 9503   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9504   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9505   attributes.set_is_evex_instruction();
 9506   attributes.set_embedded_opmask_register_specifier(mask);
 9507   if (merge) {
 9508     attributes.reset_is_clear_context();
 9509   }
 9510   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9511   emit_int16(0x51, (0xC0 | encode));
 9512 }
 9513 
 9514 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9515   InstructionMark im(this);
 9516   assert(VM_Version::supports_evex(), "");
 9517   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9518   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9519   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9520   attributes.set_is_evex_instruction();
 9521   attributes.set_embedded_opmask_register_specifier(mask);
 9522   if (merge) {
 9523     attributes.reset_is_clear_context();
 9524   }
 9525   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9526   emit_int8(0x51);
 9527   emit_operand(dst, src, 0);
 9528 }
 9529 
 9530 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9531   assert(VM_Version::supports_evex(), "");
 9532   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9533   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9534   attributes.set_is_evex_instruction();
 9535   attributes.set_embedded_opmask_register_specifier(mask);
 9536   if (merge) {
 9537     attributes.reset_is_clear_context();
 9538   }
 9539   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9540   emit_int16(0x51, (0xC0 | encode));
 9541 }
 9542 
 9543 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9544   InstructionMark im(this);
 9545   assert(VM_Version::supports_evex(), "");
 9546   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9547   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9548   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9549   attributes.set_is_evex_instruction();
 9550   attributes.set_embedded_opmask_register_specifier(mask);
 9551   if (merge) {
 9552     attributes.reset_is_clear_context();
 9553   }
 9554   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9555   emit_int8(0x51);
 9556   emit_operand(dst, src, 0);
 9557 }
 9558 
 9559 
 9560 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9561   assert(VM_Version::supports_evex(), "");
 9562   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9563   InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9564   attributes.set_is_evex_instruction();
 9565   attributes.set_embedded_opmask_register_specifier(mask);
 9566   if (merge) {
 9567     attributes.reset_is_clear_context();
 9568   }
 9569   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9570   emit_int16(0x5E, (0xC0 | encode));
 9571 }
 9572 
 9573 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9574   InstructionMark im(this);
 9575   assert(VM_Version::supports_evex(), "");
 9576   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9577   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9578   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9579   attributes.set_is_evex_instruction();
 9580   attributes.set_embedded_opmask_register_specifier(mask);
 9581   if (merge) {
 9582     attributes.reset_is_clear_context();
 9583   }
 9584   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
 9585   emit_int8(0x5E);
 9586   emit_operand(dst, src, 0);
 9587 }
 9588 
 9589 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9590   assert(VM_Version::supports_evex(), "");
 9591   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9592   InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9593   attributes.set_is_evex_instruction();
 9594   attributes.set_embedded_opmask_register_specifier(mask);
 9595   if (merge) {
 9596     attributes.reset_is_clear_context();
 9597   }
 9598   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9599   emit_int16(0x5E, (0xC0 | encode));
 9600 }
 9601 
 9602 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9603   InstructionMark im(this);
 9604   assert(VM_Version::supports_evex(), "");
 9605   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9606   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9607   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9608   attributes.set_is_evex_instruction();
 9609   attributes.set_embedded_opmask_register_specifier(mask);
 9610   if (merge) {
 9611     attributes.reset_is_clear_context();
 9612   }
 9613   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9614   emit_int8(0x5E);
 9615   emit_operand(dst, src, 0);
 9616 }
 9617 
 9618 void Assembler::evdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src, EvexRoundPrefix rmode) {
 9619   assert(VM_Version::supports_evex(), "");
 9620   InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
 9621   attributes.set_extended_context();
 9622   attributes.set_is_evex_instruction();
 9623   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
 9624   emit_int16(0x5E, (0xC0 | encode));
 9625 }
 9626 
 9627 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9628   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9629   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9630   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9631   attributes.set_is_evex_instruction();
 9632   attributes.set_embedded_opmask_register_specifier(mask);
 9633   if (merge) {
 9634     attributes.reset_is_clear_context();
 9635   }
 9636   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9637   emit_int16(0x1C, (0xC0 | encode));
 9638 }
 9639 
 9640 
 9641 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9642   InstructionMark im(this);
 9643   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9644   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9645   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9646   attributes.set_is_evex_instruction();
 9647   attributes.set_embedded_opmask_register_specifier(mask);
 9648   if (merge) {
 9649     attributes.reset_is_clear_context();
 9650   }
 9651   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9652   emit_int8(0x1C);
 9653   emit_operand(dst, src, 0);
 9654 }
 9655 
 9656 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9657   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9658   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9659   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9660   attributes.set_is_evex_instruction();
 9661   attributes.set_embedded_opmask_register_specifier(mask);
 9662   if (merge) {
 9663     attributes.reset_is_clear_context();
 9664   }
 9665   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9666   emit_int16(0x1D, (0xC0 | encode));
 9667 }
 9668 
 9669 
 9670 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9671   InstructionMark im(this);
 9672   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9673   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9674   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9675   attributes.set_is_evex_instruction();
 9676   attributes.set_embedded_opmask_register_specifier(mask);
 9677   if (merge) {
 9678     attributes.reset_is_clear_context();
 9679   }
 9680   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9681   emit_int8(0x1D);
 9682   emit_operand(dst, src, 0);
 9683 }
 9684 
 9685 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9686   assert(VM_Version::supports_evex(), "");
 9687   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9688   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9689   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9690   attributes.set_is_evex_instruction();
 9691   attributes.set_embedded_opmask_register_specifier(mask);
 9692   if (merge) {
 9693     attributes.reset_is_clear_context();
 9694   }
 9695   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9696   emit_int16(0x1E, (0xC0 | encode));
 9697 }
 9698 
 9699 
 9700 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9701   InstructionMark im(this);
 9702   assert(VM_Version::supports_evex(), "");
 9703   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9704   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9705   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9706   attributes.set_is_evex_instruction();
 9707   attributes.set_embedded_opmask_register_specifier(mask);
 9708   if (merge) {
 9709     attributes.reset_is_clear_context();
 9710   }
 9711   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9712   emit_int8(0x1E);
 9713   emit_operand(dst, src, 0);
 9714 }
 9715 
 9716 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
 9717   assert(VM_Version::supports_evex(), "");
 9718   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9719   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9720   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9721   attributes.set_is_evex_instruction();
 9722   attributes.set_embedded_opmask_register_specifier(mask);
 9723   if (merge) {
 9724     attributes.reset_is_clear_context();
 9725   }
 9726   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9727   emit_int16(0x1F, (0xC0 | encode));
 9728 }
 9729 
 9730 
 9731 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
 9732   InstructionMark im(this);
 9733   assert(VM_Version::supports_evex(), "");
 9734   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9735   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9736   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9737   attributes.set_is_evex_instruction();
 9738   attributes.set_embedded_opmask_register_specifier(mask);
 9739   if (merge) {
 9740     attributes.reset_is_clear_context();
 9741   }
 9742   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9743   emit_int8(0x1F);
 9744   emit_operand(dst, src, 0);
 9745 }
 9746 
 9747 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9748   assert(VM_Version::supports_evex(), "");
 9749   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9750   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9751   attributes.set_is_evex_instruction();
 9752   attributes.set_embedded_opmask_register_specifier(mask);
 9753   if (merge) {
 9754     attributes.reset_is_clear_context();
 9755   }
 9756   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9757   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9758 }
 9759 
 9760 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9761   InstructionMark im(this);
 9762   assert(VM_Version::supports_evex(), "");
 9763   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9764   InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9765   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9766   attributes.set_is_evex_instruction();
 9767   attributes.set_embedded_opmask_register_specifier(mask);
 9768   if (merge) {
 9769     attributes.reset_is_clear_context();
 9770   }
 9771   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9772   emit_int8((unsigned char)0xA8);
 9773   emit_operand(dst, src, 0);
 9774 }
 9775 
 9776 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9777   assert(VM_Version::supports_evex(), "");
 9778   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9779   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9780   attributes.set_is_evex_instruction();
 9781   attributes.set_embedded_opmask_register_specifier(mask);
 9782   if (merge) {
 9783     attributes.reset_is_clear_context();
 9784   }
 9785   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9786   emit_int16((unsigned char)0xA8, (0xC0 | encode));
 9787 }
 9788 
 9789 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9790   InstructionMark im(this);
 9791   assert(VM_Version::supports_evex(), "");
 9792   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9793   InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
 9794   attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
 9795   attributes.set_is_evex_instruction();
 9796   attributes.set_embedded_opmask_register_specifier(mask);
 9797   if (merge) {
 9798     attributes.reset_is_clear_context();
 9799   }
 9800   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9801   emit_int8((unsigned char)0xA8);
 9802   emit_operand(dst, src, 0);
 9803 }
 9804 
 9805 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9806   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9807   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9808   attributes.set_is_evex_instruction();
 9809   attributes.set_embedded_opmask_register_specifier(mask);
 9810   if (merge) {
 9811     attributes.reset_is_clear_context();
 9812   }
 9813   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9814   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9815 }
 9816 
 9817 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9818   assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9819   InstructionMark im(this);
 9820   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9821   attributes.set_is_evex_instruction();
 9822   attributes.set_embedded_opmask_register_specifier(mask);
 9823   if (merge) {
 9824     attributes.reset_is_clear_context();
 9825   }
 9826   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9827   emit_int8((unsigned char)0x8D);
 9828   emit_operand(dst, src, 0);
 9829 }
 9830 
 9831 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9832   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9833   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9834   attributes.set_is_evex_instruction();
 9835   attributes.set_embedded_opmask_register_specifier(mask);
 9836   if (merge) {
 9837     attributes.reset_is_clear_context();
 9838   }
 9839   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9840   emit_int16((unsigned char)0x8D, (0xC0 | encode));
 9841 }
 9842 
 9843 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9844   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9845   InstructionMark im(this);
 9846   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9847   attributes.set_is_evex_instruction();
 9848   attributes.set_embedded_opmask_register_specifier(mask);
 9849   if (merge) {
 9850     attributes.reset_is_clear_context();
 9851   }
 9852   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9853   emit_int8((unsigned char)0x8D);
 9854   emit_operand(dst, src, 0);
 9855 }
 9856 
 9857 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9858   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9859   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9860   attributes.set_is_evex_instruction();
 9861   attributes.set_embedded_opmask_register_specifier(mask);
 9862   if (merge) {
 9863     attributes.reset_is_clear_context();
 9864   }
 9865   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9866   emit_int16(0x36, (0xC0 | encode));
 9867 }
 9868 
 9869 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9870   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9871   InstructionMark im(this);
 9872   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9873   attributes.set_is_evex_instruction();
 9874   attributes.set_embedded_opmask_register_specifier(mask);
 9875   if (merge) {
 9876     attributes.reset_is_clear_context();
 9877   }
 9878   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9879   emit_int8(0x36);
 9880   emit_operand(dst, src, 0);
 9881 }
 9882 
 9883 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9884   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9885   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9886   attributes.set_is_evex_instruction();
 9887   attributes.set_embedded_opmask_register_specifier(mask);
 9888   if (merge) {
 9889     attributes.reset_is_clear_context();
 9890   }
 9891   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9892   emit_int16(0x36, (0xC0 | encode));
 9893 }
 9894 
 9895 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9896   assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
 9897   InstructionMark im(this);
 9898   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9899   attributes.set_is_evex_instruction();
 9900   attributes.set_embedded_opmask_register_specifier(mask);
 9901   if (merge) {
 9902     attributes.reset_is_clear_context();
 9903   }
 9904   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
 9905   emit_int8(0x36);
 9906   emit_operand(dst, src, 0);
 9907 }
 9908 
 9909 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9910   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9911   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9912   attributes.set_is_evex_instruction();
 9913   attributes.set_embedded_opmask_register_specifier(mask);
 9914   if (merge) {
 9915     attributes.reset_is_clear_context();
 9916   }
 9917   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9918   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9919 }
 9920 
 9921 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9922   assert(VM_Version::supports_evex(), "");
 9923   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9924   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9925   attributes.set_is_evex_instruction();
 9926   attributes.set_embedded_opmask_register_specifier(mask);
 9927   if (merge) {
 9928     attributes.reset_is_clear_context();
 9929   }
 9930   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9931   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9932 }
 9933 
 9934 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9935   assert(VM_Version::supports_evex(), "");
 9936   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9937   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9938   attributes.set_is_evex_instruction();
 9939   attributes.set_embedded_opmask_register_specifier(mask);
 9940   if (merge) {
 9941     attributes.reset_is_clear_context();
 9942   }
 9943   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9944   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9945 }
 9946 
 9947 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9948   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9949   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9950   attributes.set_is_evex_instruction();
 9951   attributes.set_embedded_opmask_register_specifier(mask);
 9952   if (merge) {
 9953     attributes.reset_is_clear_context();
 9954   }
 9955   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9956   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9957 }
 9958 
 9959 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9960   assert(VM_Version::supports_evex(), "");
 9961   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9962   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9963   attributes.set_is_evex_instruction();
 9964   attributes.set_embedded_opmask_register_specifier(mask);
 9965   if (merge) {
 9966     attributes.reset_is_clear_context();
 9967   }
 9968   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9969   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
 9970 }
 9971 
 9972 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9973   assert(VM_Version::supports_evex(), "");
 9974   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
 9975   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9976   attributes.set_is_evex_instruction();
 9977   attributes.set_embedded_opmask_register_specifier(mask);
 9978   if (merge) {
 9979     attributes.reset_is_clear_context();
 9980   }
 9981   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9982   emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
 9983 }
 9984 
 9985 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9986   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
 9987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
 9988   attributes.set_is_evex_instruction();
 9989   attributes.set_embedded_opmask_register_specifier(mask);
 9990   if (merge) {
 9991     attributes.reset_is_clear_context();
 9992   }
 9993   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
 9994   emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
 9995 }
 9996 
 9997 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
 9998   assert(VM_Version::supports_evex(), "");
 9999   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10000   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10001   attributes.set_is_evex_instruction();
10002   attributes.set_embedded_opmask_register_specifier(mask);
10003   if (merge) {
10004     attributes.reset_is_clear_context();
10005   }
10006   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10007   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
10008 }
10009 
10010 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
10011   assert(VM_Version::supports_evex(), "");
10012   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10013   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10014   attributes.set_is_evex_instruction();
10015   attributes.set_embedded_opmask_register_specifier(mask);
10016   if (merge) {
10017     attributes.reset_is_clear_context();
10018   }
10019   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10020   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
10021 }
10022 
10023 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10024   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10025   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10026   attributes.set_is_evex_instruction();
10027   attributes.set_embedded_opmask_register_specifier(mask);
10028   if (merge) {
10029     attributes.reset_is_clear_context();
10030   }
10031   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10032   emit_int16((unsigned char)0xF1, (0xC0 | encode));
10033 }
10034 
10035 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10036   assert(VM_Version::supports_evex(), "");
10037   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10038   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10039   attributes.set_is_evex_instruction();
10040   attributes.set_embedded_opmask_register_specifier(mask);
10041   if (merge) {
10042     attributes.reset_is_clear_context();
10043   }
10044   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10045   emit_int16((unsigned char)0xF2, (0xC0 | encode));
10046 }
10047 
10048 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10049   assert(VM_Version::supports_evex(), "");
10050   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10051   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10052   attributes.set_is_evex_instruction();
10053   attributes.set_embedded_opmask_register_specifier(mask);
10054   if (merge) {
10055     attributes.reset_is_clear_context();
10056   }
10057   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10058   emit_int16((unsigned char)0xF3, (0xC0 | encode));
10059 }
10060 
10061 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10062   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10063   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10064   attributes.set_is_evex_instruction();
10065   attributes.set_embedded_opmask_register_specifier(mask);
10066   if (merge) {
10067     attributes.reset_is_clear_context();
10068   }
10069   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10070   emit_int16((unsigned char)0xD1, (0xC0 | encode));
10071 }
10072 
10073 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10074   assert(VM_Version::supports_evex(), "");
10075   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10076   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10077   attributes.set_is_evex_instruction();
10078   attributes.set_embedded_opmask_register_specifier(mask);
10079   if (merge) {
10080     attributes.reset_is_clear_context();
10081   }
10082   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10083   emit_int16((unsigned char)0xD2, (0xC0 | encode));
10084 }
10085 
10086 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10087   assert(VM_Version::supports_evex(), "");
10088   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10089   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10090   attributes.set_is_evex_instruction();
10091   attributes.set_embedded_opmask_register_specifier(mask);
10092   if (merge) {
10093     attributes.reset_is_clear_context();
10094   }
10095   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10096   emit_int16((unsigned char)0xD3, (0xC0 | encode));
10097 }
10098 
10099 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10100   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10101   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10102   attributes.set_is_evex_instruction();
10103   attributes.set_embedded_opmask_register_specifier(mask);
10104   if (merge) {
10105     attributes.reset_is_clear_context();
10106   }
10107   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10108   emit_int16((unsigned char)0xE1, (0xC0 | encode));
10109 }
10110 
10111 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10112   assert(VM_Version::supports_evex(), "");
10113   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10114   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10115   attributes.set_is_evex_instruction();
10116   attributes.set_embedded_opmask_register_specifier(mask);
10117   if (merge) {
10118     attributes.reset_is_clear_context();
10119   }
10120   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10121   emit_int16((unsigned char)0xE2, (0xC0 | encode));
10122 }
10123 
10124 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10125   assert(VM_Version::supports_evex(), "");
10126   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10127   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10128   attributes.set_is_evex_instruction();
10129   attributes.set_embedded_opmask_register_specifier(mask);
10130   if (merge) {
10131     attributes.reset_is_clear_context();
10132   }
10133   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10134   emit_int16((unsigned char)0xE2, (0xC0 | encode));
10135 }
10136 
10137 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10138   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10139   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10140   attributes.set_is_evex_instruction();
10141   attributes.set_embedded_opmask_register_specifier(mask);
10142   if (merge) {
10143     attributes.reset_is_clear_context();
10144   }
10145   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10146   emit_int16(0x12, (0xC0 | encode));
10147 }
10148 
10149 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10150   assert(VM_Version::supports_evex(), "");
10151   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10152   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10153   attributes.set_is_evex_instruction();
10154   attributes.set_embedded_opmask_register_specifier(mask);
10155   if (merge) {
10156     attributes.reset_is_clear_context();
10157   }
10158   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10159   emit_int16(0x47, (0xC0 | encode));
10160 }
10161 
10162 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10163   assert(VM_Version::supports_evex(), "");
10164   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10165   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10166   attributes.set_is_evex_instruction();
10167   attributes.set_embedded_opmask_register_specifier(mask);
10168   if (merge) {
10169     attributes.reset_is_clear_context();
10170   }
10171   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10172   emit_int16(0x47, (0xC0 | encode));
10173 }
10174 
10175 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10176   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10177   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10178   attributes.set_is_evex_instruction();
10179   attributes.set_embedded_opmask_register_specifier(mask);
10180   if (merge) {
10181     attributes.reset_is_clear_context();
10182   }
10183   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10184   emit_int16(0x10, (0xC0 | encode));
10185 }
10186 
10187 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10188   assert(VM_Version::supports_evex(), "");
10189   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10190   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10191   attributes.set_is_evex_instruction();
10192   attributes.set_embedded_opmask_register_specifier(mask);
10193   if (merge) {
10194     attributes.reset_is_clear_context();
10195   }
10196   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10197   emit_int16(0x45, (0xC0 | encode));
10198 }
10199 
10200 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10201   assert(VM_Version::supports_evex(), "");
10202   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10203   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10204   attributes.set_is_evex_instruction();
10205   attributes.set_embedded_opmask_register_specifier(mask);
10206   if (merge) {
10207     attributes.reset_is_clear_context();
10208   }
10209   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10210   emit_int16(0x45, (0xC0 | encode));
10211 }
10212 
10213 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10214   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10215   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10216   attributes.set_is_evex_instruction();
10217   attributes.set_embedded_opmask_register_specifier(mask);
10218   if (merge) {
10219     attributes.reset_is_clear_context();
10220   }
10221   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10222   emit_int16(0x11, (0xC0 | encode));
10223 }
10224 
10225 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10226   assert(VM_Version::supports_evex(), "");
10227   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10228   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10229   attributes.set_is_evex_instruction();
10230   attributes.set_embedded_opmask_register_specifier(mask);
10231   if (merge) {
10232     attributes.reset_is_clear_context();
10233   }
10234   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10235   emit_int16(0x46, (0xC0 | encode));
10236 }
10237 
10238 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10239   assert(VM_Version::supports_evex(), "");
10240   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10241   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10242   attributes.set_is_evex_instruction();
10243   attributes.set_embedded_opmask_register_specifier(mask);
10244   if (merge) {
10245     attributes.reset_is_clear_context();
10246   }
10247   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10248   emit_int16(0x46, (0xC0 | encode));
10249 }
10250 
10251 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10252   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10253   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10254   attributes.set_is_evex_instruction();
10255   attributes.set_embedded_opmask_register_specifier(mask);
10256   if (merge) {
10257     attributes.reset_is_clear_context();
10258   }
10259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10260   emit_int16(0x38, (0xC0 | encode));
10261 }
10262 
10263 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10264   assert(VM_Version::supports_avx512bw(), "");
10265   InstructionMark im(this);
10266   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10267   attributes.set_is_evex_instruction();
10268   attributes.set_embedded_opmask_register_specifier(mask);
10269   if (merge) {
10270     attributes.reset_is_clear_context();
10271   }
10272   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10273   emit_int8(0x38);
10274   emit_operand(dst, src, 0);
10275 }
10276 
10277 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10278   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10279   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10280   attributes.set_is_evex_instruction();
10281   attributes.set_embedded_opmask_register_specifier(mask);
10282   if (merge) {
10283     attributes.reset_is_clear_context();
10284   }
10285   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10286   emit_int16((unsigned char)0xEA, (0xC0 | encode));
10287 }
10288 
10289 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10290   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10291   InstructionMark im(this);
10292   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10293   attributes.set_is_evex_instruction();
10294   attributes.set_embedded_opmask_register_specifier(mask);
10295   if (merge) {
10296     attributes.reset_is_clear_context();
10297   }
10298   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10299   emit_int8((unsigned char)0xEA);
10300   emit_operand(dst, src, 0);
10301 }
10302 
10303 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10304   assert(VM_Version::supports_evex(), "");
10305   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10306   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10307   attributes.set_is_evex_instruction();
10308   attributes.set_embedded_opmask_register_specifier(mask);
10309   if (merge) {
10310     attributes.reset_is_clear_context();
10311   }
10312   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10313   emit_int16(0x39, (0xC0 | encode));
10314 }
10315 
10316 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10317   assert(VM_Version::supports_evex(), "");
10318   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10319   InstructionMark im(this);
10320   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10321   attributes.set_is_evex_instruction();
10322   attributes.set_embedded_opmask_register_specifier(mask);
10323   if (merge) {
10324     attributes.reset_is_clear_context();
10325   }
10326   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10327   emit_int8(0x39);
10328   emit_operand(dst, src, 0);
10329 }
10330 
10331 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10332   assert(VM_Version::supports_evex(), "");
10333   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10334   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10335   attributes.set_is_evex_instruction();
10336   attributes.set_embedded_opmask_register_specifier(mask);
10337   if (merge) {
10338     attributes.reset_is_clear_context();
10339   }
10340   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10341   emit_int16(0x39, (0xC0 | encode));
10342 }
10343 
10344 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10345   assert(VM_Version::supports_evex(), "");
10346   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10347   InstructionMark im(this);
10348   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10349   attributes.set_is_evex_instruction();
10350   attributes.set_embedded_opmask_register_specifier(mask);
10351   if (merge) {
10352     attributes.reset_is_clear_context();
10353   }
10354   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10355   emit_int8(0x39);
10356   emit_operand(dst, src, 0);
10357 }
10358 
10359 
10360 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10361   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10363   attributes.set_is_evex_instruction();
10364   attributes.set_embedded_opmask_register_specifier(mask);
10365   if (merge) {
10366     attributes.reset_is_clear_context();
10367   }
10368   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10369   emit_int16(0x3C, (0xC0 | encode));
10370 }
10371 
10372 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10373   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10374   InstructionMark im(this);
10375   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10376   attributes.set_is_evex_instruction();
10377   attributes.set_embedded_opmask_register_specifier(mask);
10378   if (merge) {
10379     attributes.reset_is_clear_context();
10380   }
10381   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10382   emit_int8(0x3C);
10383   emit_operand(dst, src, 0);
10384 }
10385 
10386 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10387   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10389   attributes.set_is_evex_instruction();
10390   attributes.set_embedded_opmask_register_specifier(mask);
10391   if (merge) {
10392     attributes.reset_is_clear_context();
10393   }
10394   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10395   emit_int16((unsigned char)0xEE, (0xC0 | encode));
10396 }
10397 
10398 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10399   assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10400   InstructionMark im(this);
10401   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10402   attributes.set_is_evex_instruction();
10403   attributes.set_embedded_opmask_register_specifier(mask);
10404   if (merge) {
10405     attributes.reset_is_clear_context();
10406   }
10407   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10408   emit_int8((unsigned char)0xEE);
10409   emit_operand(dst, src, 0);
10410 }
10411 
10412 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10413   assert(VM_Version::supports_evex(), "");
10414   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10415   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10416   attributes.set_is_evex_instruction();
10417   attributes.set_embedded_opmask_register_specifier(mask);
10418   if (merge) {
10419     attributes.reset_is_clear_context();
10420   }
10421   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10422   emit_int16(0x3D, (0xC0 | encode));
10423 }
10424 
10425 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10426   assert(VM_Version::supports_evex(), "");
10427   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10428   InstructionMark im(this);
10429   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10430   attributes.set_is_evex_instruction();
10431   attributes.set_embedded_opmask_register_specifier(mask);
10432   if (merge) {
10433     attributes.reset_is_clear_context();
10434   }
10435   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10436   emit_int8(0x3D);
10437   emit_operand(dst, src, 0);
10438 }
10439 
10440 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10441   assert(VM_Version::supports_evex(), "");
10442   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10443   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10444   attributes.set_is_evex_instruction();
10445   attributes.set_embedded_opmask_register_specifier(mask);
10446   if (merge) {
10447     attributes.reset_is_clear_context();
10448   }
10449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10450   emit_int16(0x3D, (0xC0 | encode));
10451 }
10452 
10453 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10454   assert(VM_Version::supports_evex(), "");
10455   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10456   InstructionMark im(this);
10457   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10458   attributes.set_is_evex_instruction();
10459   attributes.set_embedded_opmask_register_specifier(mask);
10460   if (merge) {
10461     attributes.reset_is_clear_context();
10462   }
10463   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10464   emit_int8(0x3D);
10465   emit_operand(dst, src, 0);
10466 }
10467 
10468 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10469   assert(VM_Version::supports_evex(), "requires EVEX support");
10470   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10471   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10472   attributes.set_is_evex_instruction();
10473   attributes.set_embedded_opmask_register_specifier(mask);
10474   if (merge) {
10475     attributes.reset_is_clear_context();
10476   }
10477   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10478   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10479 }
10480 
10481 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10482   assert(VM_Version::supports_evex(), "requires EVEX support");
10483   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10484   assert(dst != xnoreg, "sanity");
10485   InstructionMark im(this);
10486   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10487   attributes.set_is_evex_instruction();
10488   attributes.set_embedded_opmask_register_specifier(mask);
10489   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10490   if (merge) {
10491     attributes.reset_is_clear_context();
10492   }
10493   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10494   emit_int8(0x25);
10495   emit_operand(dst, src3, 1);
10496   emit_int8(imm8);
10497 }
10498 
10499 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10500   assert(VM_Version::supports_evex(), "requires EVEX support");
10501   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10502   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10503   attributes.set_is_evex_instruction();
10504   attributes.set_embedded_opmask_register_specifier(mask);
10505   if (merge) {
10506     attributes.reset_is_clear_context();
10507   }
10508   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10509   emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10510 }
10511 
10512 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10513   assert(VM_Version::supports_evex(), "requires EVEX support");
10514   assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10515   assert(dst != xnoreg, "sanity");
10516   InstructionMark im(this);
10517   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10518   attributes.set_is_evex_instruction();
10519   attributes.set_embedded_opmask_register_specifier(mask);
10520   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10521   if (merge) {
10522     attributes.reset_is_clear_context();
10523   }
10524   vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10525   emit_int8(0x25);
10526   emit_operand(dst, src3, 1);
10527   emit_int8(imm8);
10528 }
10529 
10530 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10531   assert(VM_Version::supports_gfni(), "");
10532   assert(VM_Version::supports_sse(), "");
10533   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10535   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10536 }
10537 
10538 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10539   assert(VM_Version::supports_gfni(), "requires GFNI support");
10540   assert(VM_Version::supports_sse(), "");
10541   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10542   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10543   emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10544 }
10545 
10546 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10547 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10548   assert(UseAVX >= 2, "");
10549   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10550   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10551   emit_int16(0x58, (0xC0 | encode));
10552 }
10553 
10554 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10555   assert(VM_Version::supports_avx2(), "");
10556   assert(dst != xnoreg, "sanity");
10557   InstructionMark im(this);
10558   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10559   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10560   // swap src<->dst for encoding
10561   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10562   emit_int8(0x58);
10563   emit_operand(dst, src, 0);
10564 }
10565 
10566 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10567 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10568   assert(VM_Version::supports_avx2(), "");
10569   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10570   attributes.set_rex_vex_w_reverted();
10571   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10572   emit_int16(0x59, (0xC0 | encode));
10573 }
10574 
10575 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10576   assert(VM_Version::supports_avx2(), "");
10577   assert(dst != xnoreg, "sanity");
10578   InstructionMark im(this);
10579   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10580   attributes.set_rex_vex_w_reverted();
10581   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10582   // swap src<->dst for encoding
10583   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10584   emit_int8(0x59);
10585   emit_operand(dst, src, 0);
10586 }
10587 
10588 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10589   assert(vector_len != Assembler::AVX_128bit, "");
10590   assert(VM_Version::supports_evex(), "");
10591   assert(dst != xnoreg, "sanity");
10592   InstructionMark im(this);
10593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10594   attributes.set_rex_vex_w_reverted();
10595   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10596   // swap src<->dst for encoding
10597   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10598   emit_int8(0x5A);
10599   emit_operand(dst, src, 0);
10600 }
10601 
10602 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10603   assert(vector_len != Assembler::AVX_128bit, "");
10604   assert(VM_Version::supports_avx512dq(), "");
10605   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10606   attributes.set_rex_vex_w_reverted();
10607   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10608   emit_int16(0x5A, (0xC0 | encode));
10609 }
10610 
10611 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10612   assert(vector_len != Assembler::AVX_128bit, "");
10613   assert(VM_Version::supports_avx512dq(), "");
10614   assert(dst != xnoreg, "sanity");
10615   InstructionMark im(this);
10616   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10617   attributes.set_rex_vex_w_reverted();
10618   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10619   // swap src<->dst for encoding
10620   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10621   emit_int8(0x5A);
10622   emit_operand(dst, src, 0);
10623 }
10624 
10625 // scalar single/double precision replicate
10626 
10627 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10628 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10629   assert(VM_Version::supports_avx2(), "");
10630   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10631   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10632   emit_int16(0x18, (0xC0 | encode));
10633 }
10634 
10635 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10636   assert(VM_Version::supports_avx(), "");
10637   assert(dst != xnoreg, "sanity");
10638   InstructionMark im(this);
10639   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10640   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10641   // swap src<->dst for encoding
10642   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10643   emit_int8(0x18);
10644   emit_operand(dst, src, 0);
10645 }
10646 
10647 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10648 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10649   assert(VM_Version::supports_avx2(), "");
10650   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10651   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10652   attributes.set_rex_vex_w_reverted();
10653   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10654   emit_int16(0x19, (0xC0 | encode));
10655 }
10656 
10657 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10658   assert(VM_Version::supports_avx(), "");
10659   assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10660   assert(dst != xnoreg, "sanity");
10661   InstructionMark im(this);
10662   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10663   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10664   attributes.set_rex_vex_w_reverted();
10665   // swap src<->dst for encoding
10666   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10667   emit_int8(0x19);
10668   emit_operand(dst, src, 0);
10669 }
10670 
10671 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10672   assert(VM_Version::supports_avx(), "");
10673   assert(vector_len == AVX_256bit, "");
10674   assert(dst != xnoreg, "sanity");
10675   InstructionMark im(this);
10676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10677   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10678   // swap src<->dst for encoding
10679   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10680   emit_int8(0x1A);
10681   emit_operand(dst, src, 0);
10682 }
10683 
10684 // gpr source broadcast forms
10685 
10686 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10687 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10688   assert(VM_Version::supports_avx512bw(), "");
10689   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10690   attributes.set_is_evex_instruction();
10691   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10692   emit_int16(0x7A, (0xC0 | encode));
10693 }
10694 
10695 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10696 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10697   assert(VM_Version::supports_avx512bw(), "");
10698   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10699   attributes.set_is_evex_instruction();
10700   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10701   emit_int16(0x7B, (0xC0 | encode));
10702 }
10703 
10704 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10705 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10706   assert(VM_Version::supports_evex(), "");
10707   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10708   attributes.set_is_evex_instruction();
10709   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10710   emit_int16(0x7C, (0xC0 | encode));
10711 }
10712 
10713 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10714 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10715   assert(VM_Version::supports_evex(), "");
10716   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10717   attributes.set_is_evex_instruction();
10718   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10719   emit_int16(0x7C, (0xC0 | encode));
10720 }
10721 
10722 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10723   assert(VM_Version::supports_avx2(), "");
10724   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10725   assert(dst != xnoreg, "sanity");
10726   assert(src.isxmmindex(),"expected to be xmm index");
10727   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10728   InstructionMark im(this);
10729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10730   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10731   emit_int8((unsigned char)0x90);
10732   emit_operand(dst, src, 0);
10733 }
10734 
10735 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10736   assert(VM_Version::supports_avx2(), "");
10737   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10738   assert(dst != xnoreg, "sanity");
10739   assert(src.isxmmindex(),"expected to be xmm index");
10740   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10741   InstructionMark im(this);
10742   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10743   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10744   emit_int8((unsigned char)0x90);
10745   emit_operand(dst, src, 0);
10746 }
10747 
10748 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10749   assert(VM_Version::supports_avx2(), "");
10750   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10751   assert(dst != xnoreg, "sanity");
10752   assert(src.isxmmindex(),"expected to be xmm index");
10753   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10754   InstructionMark im(this);
10755   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10756   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10757   emit_int8((unsigned char)0x92);
10758   emit_operand(dst, src, 0);
10759 }
10760 
10761 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10762   assert(VM_Version::supports_avx2(), "");
10763   assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10764   assert(dst != xnoreg, "sanity");
10765   assert(src.isxmmindex(),"expected to be xmm index");
10766   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10767   InstructionMark im(this);
10768   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10769   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10770   emit_int8((unsigned char)0x92);
10771   emit_operand(dst, src, 0);
10772 }
10773 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10774   assert(VM_Version::supports_evex(), "");
10775   assert(dst != xnoreg, "sanity");
10776   assert(src.isxmmindex(),"expected to be xmm index");
10777   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10778   assert(mask != k0, "instruction will #UD if mask is in k0");
10779   InstructionMark im(this);
10780   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10782   attributes.reset_is_clear_context();
10783   attributes.set_embedded_opmask_register_specifier(mask);
10784   attributes.set_is_evex_instruction();
10785   // swap src<->dst for encoding
10786   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10787   emit_int8((unsigned char)0x90);
10788   emit_operand(dst, src, 0);
10789 }
10790 
10791 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10792   assert(VM_Version::supports_evex(), "");
10793   assert(dst != xnoreg, "sanity");
10794   assert(src.isxmmindex(),"expected to be xmm index");
10795   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10796   assert(mask != k0, "instruction will #UD if mask is in k0");
10797   InstructionMark im(this);
10798   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10799   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10800   attributes.reset_is_clear_context();
10801   attributes.set_embedded_opmask_register_specifier(mask);
10802   attributes.set_is_evex_instruction();
10803   // swap src<->dst for encoding
10804   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10805   emit_int8((unsigned char)0x90);
10806   emit_operand(dst, src, 0);
10807 }
10808 
10809 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10810   assert(VM_Version::supports_evex(), "");
10811   assert(dst != xnoreg, "sanity");
10812   assert(src.isxmmindex(),"expected to be xmm index");
10813   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10814   assert(mask != k0, "instruction will #UD if mask is in k0");
10815   InstructionMark im(this);
10816   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10817   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10818   attributes.reset_is_clear_context();
10819   attributes.set_embedded_opmask_register_specifier(mask);
10820   attributes.set_is_evex_instruction();
10821   // swap src<->dst for encoding
10822   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10823   emit_int8((unsigned char)0x92);
10824   emit_operand(dst, src, 0);
10825 }
10826 
10827 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10828   assert(VM_Version::supports_evex(), "");
10829   assert(dst != xnoreg, "sanity");
10830   assert(src.isxmmindex(),"expected to be xmm index");
10831   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10832   assert(mask != k0, "instruction will #UD if mask is in k0");
10833   InstructionMark im(this);
10834   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10835   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10836   attributes.reset_is_clear_context();
10837   attributes.set_embedded_opmask_register_specifier(mask);
10838   attributes.set_is_evex_instruction();
10839   // swap src<->dst for encoding
10840   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10841   emit_int8((unsigned char)0x92);
10842   emit_operand(dst, src, 0);
10843 }
10844 
10845 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10846   assert(VM_Version::supports_evex(), "");
10847   assert(mask != k0, "instruction will #UD if mask is in k0");
10848   InstructionMark im(this);
10849   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10850   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10851   attributes.reset_is_clear_context();
10852   attributes.set_embedded_opmask_register_specifier(mask);
10853   attributes.set_is_evex_instruction();
10854   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10855   emit_int8((unsigned char)0xA0);
10856   emit_operand(src, dst, 0);
10857 }
10858 
10859 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10860   assert(VM_Version::supports_evex(), "");
10861   assert(mask != k0, "instruction will #UD if mask is in k0");
10862   InstructionMark im(this);
10863   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10864   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10865   attributes.reset_is_clear_context();
10866   attributes.set_embedded_opmask_register_specifier(mask);
10867   attributes.set_is_evex_instruction();
10868   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10869   emit_int8((unsigned char)0xA0);
10870   emit_operand(src, dst, 0);
10871 }
10872 
10873 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10874   assert(VM_Version::supports_evex(), "");
10875   assert(mask != k0, "instruction will #UD if mask is in k0");
10876   InstructionMark im(this);
10877   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10878   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10879   attributes.reset_is_clear_context();
10880   attributes.set_embedded_opmask_register_specifier(mask);
10881   attributes.set_is_evex_instruction();
10882   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10883   emit_int8((unsigned char)0xA2);
10884   emit_operand(src, dst, 0);
10885 }
10886 
10887 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10888   assert(VM_Version::supports_evex(), "");
10889   assert(mask != k0, "instruction will #UD if mask is in k0");
10890   InstructionMark im(this);
10891   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10892   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10893   attributes.reset_is_clear_context();
10894   attributes.set_embedded_opmask_register_specifier(mask);
10895   attributes.set_is_evex_instruction();
10896   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10897   emit_int8((unsigned char)0xA2);
10898   emit_operand(src, dst, 0);
10899 }
10900 // Carry-Less Multiplication Quadword
10901 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10902   assert(VM_Version::supports_clmul(), "");
10903   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10904   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10905   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10906 }
10907 
10908 // Carry-Less Multiplication Quadword
10909 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10910   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10911   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10912   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10913   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10914 }
10915 
10916 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10917   assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10918   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10919   attributes.set_is_evex_instruction();
10920   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10921   emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10922 }
10923 
10924 void Assembler::vzeroupper_uncached() {
10925   if (VM_Version::supports_vzeroupper()) {
10926     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10927     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10928     emit_int8(0x77);
10929   }
10930 }
10931 
10932 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10933   // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10934   assert(VM_Version::supports_evex(), "");
10935   assert(VM_Version::supports_avx512dq(), "");
10936   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10937   attributes.set_is_evex_instruction();
10938   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10939   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10940 }
10941 
10942 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10943   // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10944   assert(VM_Version::supports_evex(), "");
10945   assert(VM_Version::supports_avx512dq(), "");
10946   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10947   attributes.set_is_evex_instruction();
10948   int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10949   emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10950 }
10951 
10952 void Assembler::fld_x(Address adr) {
10953   InstructionMark im(this);
10954   emit_int8((unsigned char)0xDB);
10955   emit_operand32(rbp, adr, 0);
10956 }
10957 
10958 void Assembler::fstp_x(Address adr) {
10959   InstructionMark im(this);
10960   emit_int8((unsigned char)0xDB);
10961   emit_operand32(rdi, adr, 0);
10962 }
10963 
10964 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10965   assert(reg->encoding() < 8, "no extended registers");
10966   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10967   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10968 }
10969 
10970 #ifndef _LP64
10971 // 32bit only pieces of the assembler
10972 
10973 void Assembler::emms() {
10974   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10975   emit_int16(0x0F, 0x77);
10976 }
10977 
10978 void Assembler::vzeroupper() {
10979   vzeroupper_uncached();
10980 }
10981 
10982 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10983   // NO PREFIX AS NEVER 64BIT
10984   InstructionMark im(this);
10985   emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10986   emit_data(imm32, rspec, 0);
10987 }
10988 
10989 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10990   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10991   InstructionMark im(this);
10992   emit_int8((unsigned char)0x81);
10993   emit_operand(rdi, src1, 4);
10994   emit_data(imm32, rspec, 0);
10995 }
10996 
10997 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10998 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10999 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
11000 void Assembler::cmpxchg8(Address adr) {
11001   InstructionMark im(this);
11002   emit_int16(0x0F, (unsigned char)0xC7);
11003   emit_operand(rcx, adr, 0);
11004 }
11005 
11006 void Assembler::decl(Register dst) {
11007   // Don't use it directly. Use MacroAssembler::decrementl() instead.
11008  emit_int8(0x48 | dst->encoding());
11009 }
11010 
11011 // 64bit doesn't use the x87
11012 
11013 void Assembler::emit_farith(int b1, int b2, int i) {
11014   assert(isByte(b1) && isByte(b2), "wrong opcode");
11015   assert(0 <= i &&  i < 8, "illegal stack offset");
11016   emit_int16(b1, b2 + i);
11017 }
11018 
11019 void Assembler::fabs() {
11020   emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
11021 }
11022 
11023 void Assembler::fadd(int i) {
11024   emit_farith(0xD8, 0xC0, i);
11025 }
11026 
11027 void Assembler::fadd_d(Address src) {
11028   InstructionMark im(this);
11029   emit_int8((unsigned char)0xDC);
11030   emit_operand32(rax, src, 0);
11031 }
11032 
11033 void Assembler::fadd_s(Address src) {
11034   InstructionMark im(this);
11035   emit_int8((unsigned char)0xD8);
11036   emit_operand32(rax, src, 0);
11037 }
11038 
11039 void Assembler::fadda(int i) {
11040   emit_farith(0xDC, 0xC0, i);
11041 }
11042 
11043 void Assembler::faddp(int i) {
11044   emit_farith(0xDE, 0xC0, i);
11045 }
11046 
11047 void Assembler::fchs() {
11048   emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
11049 }
11050 
11051 void Assembler::fcom(int i) {
11052   emit_farith(0xD8, 0xD0, i);
11053 }
11054 
11055 void Assembler::fcomp(int i) {
11056   emit_farith(0xD8, 0xD8, i);
11057 }
11058 
11059 void Assembler::fcomp_d(Address src) {
11060   InstructionMark im(this);
11061   emit_int8((unsigned char)0xDC);
11062   emit_operand32(rbx, src, 0);
11063 }
11064 
11065 void Assembler::fcomp_s(Address src) {
11066   InstructionMark im(this);
11067   emit_int8((unsigned char)0xD8);
11068   emit_operand32(rbx, src, 0);
11069 }
11070 
11071 void Assembler::fcompp() {
11072   emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
11073 }
11074 
11075 void Assembler::fcos() {
11076   emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
11077 }
11078 
11079 void Assembler::fdecstp() {
11080   emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
11081 }
11082 
11083 void Assembler::fdiv(int i) {
11084   emit_farith(0xD8, 0xF0, i);
11085 }
11086 
11087 void Assembler::fdiv_d(Address src) {
11088   InstructionMark im(this);
11089   emit_int8((unsigned char)0xDC);
11090   emit_operand32(rsi, src, 0);
11091 }
11092 
11093 void Assembler::fdiv_s(Address src) {
11094   InstructionMark im(this);
11095   emit_int8((unsigned char)0xD8);
11096   emit_operand32(rsi, src, 0);
11097 }
11098 
11099 void Assembler::fdiva(int i) {
11100   emit_farith(0xDC, 0xF8, i);
11101 }
11102 
11103 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
11104 //       is erroneous for some of the floating-point instructions below.
11105 
11106 void Assembler::fdivp(int i) {
11107   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
11108 }
11109 
11110 void Assembler::fdivr(int i) {
11111   emit_farith(0xD8, 0xF8, i);
11112 }
11113 
11114 void Assembler::fdivr_d(Address src) {
11115   InstructionMark im(this);
11116   emit_int8((unsigned char)0xDC);
11117   emit_operand32(rdi, src, 0);
11118 }
11119 
11120 void Assembler::fdivr_s(Address src) {
11121   InstructionMark im(this);
11122   emit_int8((unsigned char)0xD8);
11123   emit_operand32(rdi, src, 0);
11124 }
11125 
11126 void Assembler::fdivra(int i) {
11127   emit_farith(0xDC, 0xF0, i);
11128 }
11129 
11130 void Assembler::fdivrp(int i) {
11131   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
11132 }
11133 
11134 void Assembler::ffree(int i) {
11135   emit_farith(0xDD, 0xC0, i);
11136 }
11137 
11138 void Assembler::fild_d(Address adr) {
11139   InstructionMark im(this);
11140   emit_int8((unsigned char)0xDF);
11141   emit_operand32(rbp, adr, 0);
11142 }
11143 
11144 void Assembler::fild_s(Address adr) {
11145   InstructionMark im(this);
11146   emit_int8((unsigned char)0xDB);
11147   emit_operand32(rax, adr, 0);
11148 }
11149 
11150 void Assembler::fincstp() {
11151   emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
11152 }
11153 
11154 void Assembler::finit() {
11155   emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
11156 }
11157 
11158 void Assembler::fist_s(Address adr) {
11159   InstructionMark im(this);
11160   emit_int8((unsigned char)0xDB);
11161   emit_operand32(rdx, adr, 0);
11162 }
11163 
11164 void Assembler::fistp_d(Address adr) {
11165   InstructionMark im(this);
11166   emit_int8((unsigned char)0xDF);
11167   emit_operand32(rdi, adr, 0);
11168 }
11169 
11170 void Assembler::fistp_s(Address adr) {
11171   InstructionMark im(this);
11172   emit_int8((unsigned char)0xDB);
11173   emit_operand32(rbx, adr, 0);
11174 }
11175 
11176 void Assembler::fld1() {
11177   emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
11178 }
11179 
11180 void Assembler::fld_d(Address adr) {
11181   InstructionMark im(this);
11182   emit_int8((unsigned char)0xDD);
11183   emit_operand32(rax, adr, 0);
11184 }
11185 
11186 void Assembler::fld_s(Address adr) {
11187   InstructionMark im(this);
11188   emit_int8((unsigned char)0xD9);
11189   emit_operand32(rax, adr, 0);
11190 }
11191 
11192 
11193 void Assembler::fld_s(int index) {
11194   emit_farith(0xD9, 0xC0, index);
11195 }
11196 
11197 void Assembler::fldcw(Address src) {
11198   InstructionMark im(this);
11199   emit_int8((unsigned char)0xD9);
11200   emit_operand32(rbp, src, 0);
11201 }
11202 
11203 void Assembler::fldenv(Address src) {
11204   InstructionMark im(this);
11205   emit_int8((unsigned char)0xD9);
11206   emit_operand32(rsp, src, 0);
11207 }
11208 
11209 void Assembler::fldlg2() {
11210   emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
11211 }
11212 
11213 void Assembler::fldln2() {
11214   emit_int16((unsigned char)0xD9, (unsigned char)0xED);
11215 }
11216 
11217 void Assembler::fldz() {
11218   emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
11219 }
11220 
11221 void Assembler::flog() {
11222   fldln2();
11223   fxch();
11224   fyl2x();
11225 }
11226 
11227 void Assembler::flog10() {
11228   fldlg2();
11229   fxch();
11230   fyl2x();
11231 }
11232 
11233 void Assembler::fmul(int i) {
11234   emit_farith(0xD8, 0xC8, i);
11235 }
11236 
11237 void Assembler::fmul_d(Address src) {
11238   InstructionMark im(this);
11239   emit_int8((unsigned char)0xDC);
11240   emit_operand32(rcx, src, 0);
11241 }
11242 
11243 void Assembler::fmul_s(Address src) {
11244   InstructionMark im(this);
11245   emit_int8((unsigned char)0xD8);
11246   emit_operand32(rcx, src, 0);
11247 }
11248 
11249 void Assembler::fmula(int i) {
11250   emit_farith(0xDC, 0xC8, i);
11251 }
11252 
11253 void Assembler::fmulp(int i) {
11254   emit_farith(0xDE, 0xC8, i);
11255 }
11256 
11257 void Assembler::fnsave(Address dst) {
11258   InstructionMark im(this);
11259   emit_int8((unsigned char)0xDD);
11260   emit_operand32(rsi, dst, 0);
11261 }
11262 
11263 void Assembler::fnstcw(Address src) {
11264   InstructionMark im(this);
11265   emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
11266   emit_operand32(rdi, src, 0);
11267 }
11268 
11269 void Assembler::fnstsw_ax() {
11270   emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
11271 }
11272 
11273 void Assembler::fprem() {
11274   emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
11275 }
11276 
11277 void Assembler::fprem1() {
11278   emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
11279 }
11280 
11281 void Assembler::frstor(Address src) {
11282   InstructionMark im(this);
11283   emit_int8((unsigned char)0xDD);
11284   emit_operand32(rsp, src, 0);
11285 }
11286 
11287 void Assembler::fsin() {
11288   emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
11289 }
11290 
11291 void Assembler::fsqrt() {
11292   emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11293 }
11294 
11295 void Assembler::fst_d(Address adr) {
11296   InstructionMark im(this);
11297   emit_int8((unsigned char)0xDD);
11298   emit_operand32(rdx, adr, 0);
11299 }
11300 
11301 void Assembler::fst_s(Address adr) {
11302   InstructionMark im(this);
11303   emit_int8((unsigned char)0xD9);
11304   emit_operand32(rdx, adr, 0);
11305 }
11306 
11307 void Assembler::fstp_d(Address adr) {
11308   InstructionMark im(this);
11309   emit_int8((unsigned char)0xDD);
11310   emit_operand32(rbx, adr, 0);
11311 }
11312 
11313 void Assembler::fstp_d(int index) {
11314   emit_farith(0xDD, 0xD8, index);
11315 }
11316 
11317 void Assembler::fstp_s(Address adr) {
11318   InstructionMark im(this);
11319   emit_int8((unsigned char)0xD9);
11320   emit_operand32(rbx, adr, 0);
11321 }
11322 
11323 void Assembler::fsub(int i) {
11324   emit_farith(0xD8, 0xE0, i);
11325 }
11326 
11327 void Assembler::fsub_d(Address src) {
11328   InstructionMark im(this);
11329   emit_int8((unsigned char)0xDC);
11330   emit_operand32(rsp, src, 0);
11331 }
11332 
11333 void Assembler::fsub_s(Address src) {
11334   InstructionMark im(this);
11335   emit_int8((unsigned char)0xD8);
11336   emit_operand32(rsp, src, 0);
11337 }
11338 
11339 void Assembler::fsuba(int i) {
11340   emit_farith(0xDC, 0xE8, i);
11341 }
11342 
11343 void Assembler::fsubp(int i) {
11344   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11345 }
11346 
11347 void Assembler::fsubr(int i) {
11348   emit_farith(0xD8, 0xE8, i);
11349 }
11350 
11351 void Assembler::fsubr_d(Address src) {
11352   InstructionMark im(this);
11353   emit_int8((unsigned char)0xDC);
11354   emit_operand32(rbp, src, 0);
11355 }
11356 
11357 void Assembler::fsubr_s(Address src) {
11358   InstructionMark im(this);
11359   emit_int8((unsigned char)0xD8);
11360   emit_operand32(rbp, src, 0);
11361 }
11362 
11363 void Assembler::fsubra(int i) {
11364   emit_farith(0xDC, 0xE0, i);
11365 }
11366 
11367 void Assembler::fsubrp(int i) {
11368   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11369 }
11370 
11371 void Assembler::ftan() {
11372   emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11373 }
11374 
11375 void Assembler::ftst() {
11376   emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11377 }
11378 
11379 void Assembler::fucomi(int i) {
11380   // make sure the instruction is supported (introduced for P6, together with cmov)
11381   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11382   emit_farith(0xDB, 0xE8, i);
11383 }
11384 
11385 void Assembler::fucomip(int i) {
11386   // make sure the instruction is supported (introduced for P6, together with cmov)
11387   guarantee(VM_Version::supports_cmov(), "illegal instruction");
11388   emit_farith(0xDF, 0xE8, i);
11389 }
11390 
11391 void Assembler::fwait() {
11392   emit_int8((unsigned char)0x9B);
11393 }
11394 
11395 void Assembler::fxch(int i) {
11396   emit_farith(0xD9, 0xC8, i);
11397 }
11398 
11399 void Assembler::fyl2x() {
11400   emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11401 }
11402 
11403 void Assembler::frndint() {
11404   emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11405 }
11406 
11407 void Assembler::f2xm1() {
11408   emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11409 }
11410 
11411 void Assembler::fldl2e() {
11412   emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11413 }
11414 #endif // !_LP64
11415 
11416 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11417 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11418 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11419 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
11420 
11421 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11422 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11423   if (pre > 0) {
11424     emit_int8(simd_pre[pre]);
11425   }
11426   if (rex_w) {
11427     prefixq(adr, xreg);
11428   } else {
11429     prefix(adr, xreg);
11430   }
11431   if (opc > 0) {
11432     emit_int8(0x0F);
11433     int opc2 = simd_opc[opc];
11434     if (opc2 > 0) {
11435       emit_int8(opc2);
11436     }
11437   }
11438 }
11439 
11440 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11441   if (pre > 0) {
11442     emit_int8(simd_pre[pre]);
11443   }
11444   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11445   if (opc > 0) {
11446     emit_int8(0x0F);
11447     int opc2 = simd_opc[opc];
11448     if (opc2 > 0) {
11449       emit_int8(opc2);
11450     }
11451   }
11452   return encode;
11453 }
11454 
11455 
11456 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11457   int vector_len = _attributes->get_vector_len();
11458   bool vex_w = _attributes->is_rex_vex_w();
11459   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11460     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11461     byte1 = (~byte1) & 0xE0;
11462     byte1 |= opc;
11463 
11464     int byte2 = ((~nds_enc) & 0xf) << 3;
11465     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11466 
11467     emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11468   } else {
11469     int byte1 = vex_r ? VEX_R : 0;
11470     byte1 = (~byte1) & 0x80;
11471     byte1 |= ((~nds_enc) & 0xf) << 3;
11472     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11473     emit_int16((unsigned char)VEX_2bytes, byte1);
11474   }
11475 }
11476 
11477 // This is a 4 byte encoding
11478 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11479   // EVEX 0x62 prefix
11480   // byte1 = EVEX_4bytes;
11481 
11482   bool vex_w = _attributes->is_rex_vex_w();
11483   int evex_encoding = (vex_w ? VEX_W : 0);
11484   // EVEX.b is not currently used for broadcast of single element or data rounding modes
11485   _attributes->set_evex_encoding(evex_encoding);
11486 
11487   // P0: byte 2, initialized to RXBR`00mm
11488   // instead of not'd
11489   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11490   byte2 = (~byte2) & 0xF0;
11491   // confine opc opcode extensions in mm bits to lower two bits
11492   // of form {0F, 0F_38, 0F_3A}
11493   byte2 |= opc;
11494 
11495   // P1: byte 3 as Wvvvv1pp
11496   int byte3 = ((~nds_enc) & 0xf) << 3;
11497   // p[10] is always 1
11498   byte3 |= EVEX_F;
11499   byte3 |= (vex_w & 1) << 7;
11500   // confine pre opcode extensions in pp bits to lower two bits
11501   // of form {66, F3, F2}
11502   byte3 |= pre;
11503 
11504   // P2: byte 4 as zL'Lbv'aaa
11505   // kregs are implemented in the low 3 bits as aaa
11506   int byte4 = (_attributes->is_no_reg_mask()) ?
11507               0 :
11508               _attributes->get_embedded_opmask_register_specifier();
11509   // EVEX.v` for extending EVEX.vvvv or VIDX
11510   byte4 |= (evex_v ? 0: EVEX_V);
11511   // third EXEC.b for broadcast actions
11512   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11513   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11514   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11515   // last is EVEX.z for zero/merge actions
11516   if (_attributes->is_no_reg_mask() == false &&
11517       _attributes->get_embedded_opmask_register_specifier() != 0) {
11518     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11519   }
11520 
11521   emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11522 }
11523 
11524 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11525   bool vex_r = (xreg_enc & 8) == 8;
11526   bool vex_b = adr.base_needs_rex();
11527   bool vex_x;
11528   if (adr.isxmmindex()) {
11529     vex_x = adr.xmmindex_needs_rex();
11530   } else {
11531     vex_x = adr.index_needs_rex();
11532   }
11533   set_attributes(attributes);
11534 
11535   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11536   // is allowed in legacy mode and has resources which will fit in it.
11537   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11538   if (!attributes->is_legacy_mode()) {
11539     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11540       if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11541           attributes->set_is_legacy_mode();
11542       }
11543     }
11544   }
11545 
11546   if (UseAVX > 2) {
11547     assert(((!attributes->uses_vl()) ||
11548             (attributes->get_vector_len() == AVX_512bit) ||
11549             (!_legacy_mode_vl) ||
11550             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11551     assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11552   }
11553 
11554   clear_managed();
11555   if (UseAVX > 2 && !attributes->is_legacy_mode())
11556   {
11557     bool evex_r = (xreg_enc >= 16);
11558     bool evex_v;
11559     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11560     if (adr.isxmmindex())  {
11561       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11562     } else {
11563       evex_v = (nds_enc >= 16);
11564     }
11565     attributes->set_is_evex_instruction();
11566     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11567   } else {
11568     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11569       attributes->set_rex_vex_w(false);
11570     }
11571     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11572   }
11573 }
11574 
11575 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11576   bool vex_r = (dst_enc & 8) == 8;
11577   bool vex_b = (src_enc & 8) == 8;
11578   bool vex_x = false;
11579   set_attributes(attributes);
11580 
11581   // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11582   // is allowed in legacy mode and has resources which will fit in it.
11583   // Pure EVEX instructions will have is_evex_instruction set in their definition.
11584   if (!attributes->is_legacy_mode()) {
11585     if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11586       if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11587           (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11588           attributes->set_is_legacy_mode();
11589       }
11590     }
11591   }
11592 
11593   if (UseAVX > 2) {
11594     // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11595     // Instruction with uses_vl true are vector instructions
11596     // All the vector instructions with AVX_512bit length can have legacy_mode as false
11597     // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11598     // Rest all should have legacy_mode set as true
11599     assert(((!attributes->uses_vl()) ||
11600             (attributes->get_vector_len() == AVX_512bit) ||
11601             (!_legacy_mode_vl) ||
11602             (attributes->is_legacy_mode())),"XMM register should be 0-15");
11603     // Instruction with legacy_mode true should have dst, nds and src < 15
11604     assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11605   }
11606 
11607   clear_managed();
11608   if (UseAVX > 2 && !attributes->is_legacy_mode())
11609   {
11610     bool evex_r = (dst_enc >= 16);
11611     bool evex_v = (nds_enc >= 16);
11612     // can use vex_x as bank extender on rm encoding
11613     vex_x = (src_enc >= 16);
11614     attributes->set_is_evex_instruction();
11615     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11616   } else {
11617     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11618       attributes->set_rex_vex_w(false);
11619     }
11620     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11621   }
11622 
11623   // return modrm byte components for operands
11624   return (((dst_enc & 7) << 3) | (src_enc & 7));
11625 }
11626 
11627 
11628 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11629                             VexOpcode opc, InstructionAttr *attributes) {
11630   if (UseAVX > 0) {
11631     int xreg_enc = xreg->encoding();
11632     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11633     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11634   } else {
11635     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11636     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11637   }
11638 }
11639 
11640 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11641                                       VexOpcode opc, InstructionAttr *attributes) {
11642   int dst_enc = dst->encoding();
11643   int src_enc = src->encoding();
11644   if (UseAVX > 0) {
11645     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11646     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11647   } else {
11648     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11649     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11650   }
11651 }
11652 
11653 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11654   assert(VM_Version::supports_avx(), "");
11655   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11656   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11657   emit_int16(0x5F, (0xC0 | encode));
11658 }
11659 
11660 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11661   assert(VM_Version::supports_avx(), "");
11662   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11663   attributes.set_rex_vex_w_reverted();
11664   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11665   emit_int16(0x5F, (0xC0 | encode));
11666 }
11667 
11668 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11669   assert(VM_Version::supports_avx(), "");
11670   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11671   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11672   emit_int16(0x5D, (0xC0 | encode));
11673 }
11674 
11675 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11676   assert(VM_Version::supports_avx(), "");
11677   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11678   attributes.set_rex_vex_w_reverted();
11679   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11680   emit_int16(0x5D, (0xC0 | encode));
11681 }
11682 
11683 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11684   assert(VM_Version::supports_avx(), "");
11685   assert(vector_len <= AVX_256bit, "");
11686   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11687   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11688   emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11689 }
11690 
11691 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11692   assert(VM_Version::supports_avx(), "");
11693   assert(vector_len <= AVX_256bit, "");
11694   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11695   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11696   int src2_enc = src2->encoding();
11697   emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11698 }
11699 
11700 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11701   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11702   assert(vector_len <= AVX_256bit, "");
11703   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11705   int src2_enc = src2->encoding();
11706   emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11707 }
11708 
11709 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11710   assert(VM_Version::supports_avx2(), "");
11711   assert(vector_len <= AVX_256bit, "");
11712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11713   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11714   emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11715 }
11716 
11717 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11718   assert(VM_Version::supports_avx(), "");
11719   assert(vector_len <= AVX_256bit, "");
11720   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11721   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11722   emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11723 }
11724 
11725 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11726                         ComparisonPredicateFP comparison, int vector_len) {
11727   assert(VM_Version::supports_evex(), "");
11728   // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11730   attributes.set_is_evex_instruction();
11731   attributes.set_embedded_opmask_register_specifier(mask);
11732   attributes.reset_is_clear_context();
11733   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11734   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11735 }
11736 
11737 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11738                         ComparisonPredicateFP comparison, int vector_len) {
11739   assert(VM_Version::supports_evex(), "");
11740   // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11741   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11742   attributes.set_is_evex_instruction();
11743   attributes.set_embedded_opmask_register_specifier(mask);
11744   attributes.reset_is_clear_context();
11745   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11746   emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11747 }
11748 
11749 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11750   assert(VM_Version::supports_sse4_1(), "");
11751   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11752   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11753   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11754   emit_int16(0x14, (0xC0 | encode));
11755 }
11756 
11757 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11758   assert(VM_Version::supports_sse4_1(), "");
11759   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11760   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11761   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11762   emit_int16(0x15, (0xC0 | encode));
11763 }
11764 
11765 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11766   assert(VM_Version::supports_sse4_1(), "");
11767   assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11769   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11770   emit_int16(0x10, (0xC0 | encode));
11771 }
11772 
11773 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11774   assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11775   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11776   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11777   int src2_enc = src2->encoding();
11778   emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11779 }
11780 
11781 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11782   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11783   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11784   emit_int24(0x0C, (0xC0 | encode), imm8);
11785 }
11786 
11787 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11788   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11789   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11790   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11792   emit_int16(0x64, (0xC0 | encode));
11793 }
11794 
11795 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11796   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11797   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11798   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11799   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11800   emit_int16(0x65, (0xC0 | encode));
11801 }
11802 
11803 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11804   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11805   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11806   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11807   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11808   emit_int16(0x66, (0xC0 | encode));
11809 }
11810 
11811 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11812   assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11813   assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11814   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11815   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11816   emit_int16(0x37, (0xC0 | encode));
11817 }
11818 
11819 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11820                         int comparison, bool is_signed, int vector_len) {
11821   assert(VM_Version::supports_evex(), "");
11822   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11823   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11824   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11825   attributes.set_is_evex_instruction();
11826   attributes.set_embedded_opmask_register_specifier(mask);
11827   attributes.reset_is_clear_context();
11828   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11829   int opcode = is_signed ? 0x1F : 0x1E;
11830   emit_int24(opcode, (0xC0 | encode), comparison);
11831 }
11832 
11833 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11834                         int comparison, bool is_signed, int vector_len) {
11835   assert(VM_Version::supports_evex(), "");
11836   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11837   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11838   InstructionMark im(this);
11839   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11840   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11841   attributes.set_is_evex_instruction();
11842   attributes.set_embedded_opmask_register_specifier(mask);
11843   attributes.reset_is_clear_context();
11844   int dst_enc = kdst->encoding();
11845   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11846   int opcode = is_signed ? 0x1F : 0x1E;
11847   emit_int8((unsigned char)opcode);
11848   emit_operand(as_Register(dst_enc), src, 1);
11849   emit_int8((unsigned char)comparison);
11850 }
11851 
11852 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11853                         int comparison, bool is_signed, int vector_len) {
11854   assert(VM_Version::supports_evex(), "");
11855   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11856   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11857   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11858   attributes.set_is_evex_instruction();
11859   attributes.set_embedded_opmask_register_specifier(mask);
11860   attributes.reset_is_clear_context();
11861   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11862   int opcode = is_signed ? 0x1F : 0x1E;
11863   emit_int24(opcode, (0xC0 | encode), comparison);
11864 }
11865 
11866 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11867                         int comparison, bool is_signed, int vector_len) {
11868   assert(VM_Version::supports_evex(), "");
11869   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11870   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11871   InstructionMark im(this);
11872   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11873   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11874   attributes.set_is_evex_instruction();
11875   attributes.set_embedded_opmask_register_specifier(mask);
11876   attributes.reset_is_clear_context();
11877   int dst_enc = kdst->encoding();
11878   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11879   int opcode = is_signed ? 0x1F : 0x1E;
11880   emit_int8((unsigned char)opcode);
11881   emit_operand(as_Register(dst_enc), src, 1);
11882   emit_int8((unsigned char)comparison);
11883 }
11884 
11885 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11886                         int comparison, bool is_signed, int vector_len) {
11887   assert(VM_Version::supports_evex(), "");
11888   assert(VM_Version::supports_avx512bw(), "");
11889   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11890   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11891   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11892   attributes.set_is_evex_instruction();
11893   attributes.set_embedded_opmask_register_specifier(mask);
11894   attributes.reset_is_clear_context();
11895   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11896   int opcode = is_signed ? 0x3F : 0x3E;
11897   emit_int24(opcode, (0xC0 | encode), comparison);
11898 }
11899 
11900 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11901                         int comparison, bool is_signed, int vector_len) {
11902   assert(VM_Version::supports_evex(), "");
11903   assert(VM_Version::supports_avx512bw(), "");
11904   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11905   // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11906   InstructionMark im(this);
11907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11908   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11909   attributes.set_is_evex_instruction();
11910   attributes.set_embedded_opmask_register_specifier(mask);
11911   attributes.reset_is_clear_context();
11912   int dst_enc = kdst->encoding();
11913   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11914   int opcode = is_signed ? 0x3F : 0x3E;
11915   emit_int8((unsigned char)opcode);
11916   emit_operand(as_Register(dst_enc), src, 1);
11917   emit_int8((unsigned char)comparison);
11918 }
11919 
11920 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11921                         int comparison, bool is_signed, int vector_len) {
11922   assert(VM_Version::supports_evex(), "");
11923   assert(VM_Version::supports_avx512bw(), "");
11924   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11925   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11926   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11927   attributes.set_is_evex_instruction();
11928   attributes.set_embedded_opmask_register_specifier(mask);
11929   attributes.reset_is_clear_context();
11930   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11931   int opcode = is_signed ? 0x3F : 0x3E;
11932   emit_int24(opcode, (0xC0 | encode), comparison);
11933 }
11934 
11935 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11936                         int comparison, bool is_signed, int vector_len) {
11937   assert(VM_Version::supports_evex(), "");
11938   assert(VM_Version::supports_avx512bw(), "");
11939   assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11940   // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11941   InstructionMark im(this);
11942   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11943   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11944   attributes.set_is_evex_instruction();
11945   attributes.set_embedded_opmask_register_specifier(mask);
11946   attributes.reset_is_clear_context();
11947   int dst_enc = kdst->encoding();
11948   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11949   int opcode = is_signed ? 0x3F : 0x3E;
11950   emit_int8((unsigned char)opcode);
11951   emit_operand(as_Register(dst_enc), src, 1);
11952   emit_int8((unsigned char)comparison);
11953 }
11954 
11955 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11956   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11957   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11958   attributes.set_is_evex_instruction();
11959   attributes.set_embedded_opmask_register_specifier(mask);
11960   if (merge) {
11961     attributes.reset_is_clear_context();
11962   }
11963   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11964   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11965 }
11966 
11967 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11968   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11969   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11970   attributes.set_is_evex_instruction();
11971   attributes.set_embedded_opmask_register_specifier(mask);
11972   if (merge) {
11973     attributes.reset_is_clear_context();
11974   }
11975   int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11976   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11977 }
11978 
11979 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11980   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11981   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11982   attributes.set_is_evex_instruction();
11983   attributes.set_embedded_opmask_register_specifier(mask);
11984   if (merge) {
11985     attributes.reset_is_clear_context();
11986   }
11987   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11988   emit_int16(0x14, (0xC0 | encode));
11989 }
11990 
11991 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11992   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11993   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11994   attributes.set_is_evex_instruction();
11995   attributes.set_embedded_opmask_register_specifier(mask);
11996   if (merge) {
11997     attributes.reset_is_clear_context();
11998   }
11999   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12000   emit_int16(0x14, (0xC0 | encode));
12001 }
12002 
12003 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
12004   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12005   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12006   attributes.set_is_evex_instruction();
12007   attributes.set_embedded_opmask_register_specifier(mask);
12008   if (merge) {
12009     attributes.reset_is_clear_context();
12010   }
12011   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12012   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
12013 }
12014 
12015 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
12016   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12017   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12018   attributes.set_is_evex_instruction();
12019   attributes.set_embedded_opmask_register_specifier(mask);
12020   if (merge) {
12021     attributes.reset_is_clear_context();
12022   }
12023   int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12024   emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
12025 }
12026 
12027 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12028   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12029   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12030   attributes.set_is_evex_instruction();
12031   attributes.set_embedded_opmask_register_specifier(mask);
12032   if (merge) {
12033     attributes.reset_is_clear_context();
12034   }
12035   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12036   emit_int16(0x15, (0xC0 | encode));
12037 }
12038 
12039 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12040   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12041   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12042   attributes.set_is_evex_instruction();
12043   attributes.set_embedded_opmask_register_specifier(mask);
12044   if (merge) {
12045     attributes.reset_is_clear_context();
12046   }
12047   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12048   emit_int16(0x15, (0xC0 | encode));
12049 }
12050 
12051 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
12052   assert(VM_Version::supports_avx(), "");
12053   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12054   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12055   int mask_enc = mask->encoding();
12056   emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
12057 }
12058 
12059 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12060   assert(VM_Version::supports_evex(), "");
12061   // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
12062   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12063   attributes.set_is_evex_instruction();
12064   attributes.set_embedded_opmask_register_specifier(mask);
12065   if (merge) {
12066     attributes.reset_is_clear_context();
12067   }
12068   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12069   emit_int16(0x65, (0xC0 | encode));
12070 }
12071 
12072 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12073   assert(VM_Version::supports_evex(), "");
12074   // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
12075   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12076   attributes.set_is_evex_instruction();
12077   attributes.set_embedded_opmask_register_specifier(mask);
12078   if (merge) {
12079     attributes.reset_is_clear_context();
12080   }
12081   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12082   emit_int16(0x65, (0xC0 | encode));
12083 }
12084 
12085 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12086   assert(VM_Version::supports_evex(), "");
12087   assert(VM_Version::supports_avx512bw(), "");
12088   // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
12089   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
12090   attributes.set_is_evex_instruction();
12091   attributes.set_embedded_opmask_register_specifier(mask);
12092   if (merge) {
12093     attributes.reset_is_clear_context();
12094   }
12095   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12096   emit_int16(0x66, (0xC0 | encode));
12097 }
12098 
12099 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12100   assert(VM_Version::supports_evex(), "");
12101   assert(VM_Version::supports_avx512bw(), "");
12102   // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
12103   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
12104   attributes.set_is_evex_instruction();
12105   attributes.set_embedded_opmask_register_specifier(mask);
12106   if (merge) {
12107     attributes.reset_is_clear_context();
12108   }
12109   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12110   emit_int16(0x66, (0xC0 | encode));
12111 }
12112 
12113 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12114   assert(VM_Version::supports_evex(), "");
12115   //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
12116   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12117   attributes.set_is_evex_instruction();
12118   attributes.set_embedded_opmask_register_specifier(mask);
12119   if (merge) {
12120     attributes.reset_is_clear_context();
12121   }
12122   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12123   emit_int16(0x64, (0xC0 | encode));
12124 }
12125 
12126 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12127   assert(VM_Version::supports_evex(), "");
12128   //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
12129   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12130   attributes.set_is_evex_instruction();
12131   attributes.set_embedded_opmask_register_specifier(mask);
12132   if (merge) {
12133     attributes.reset_is_clear_context();
12134   }
12135   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12136   emit_int16(0x64, (0xC0 | encode));
12137 }
12138 
12139 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
12140   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12141   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12142   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12143   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12144 }
12145 
12146 void Assembler::pextl(Register dst, Register src1, Register src2) {
12147   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12148   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12149   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12150   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12151 }
12152 
12153 void Assembler::pdepl(Register dst, Register src1, Register src2) {
12154   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12155   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12156   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12157   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12158 }
12159 
12160 void Assembler::pextq(Register dst, Register src1, Register src2) {
12161   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12162   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12163   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12164   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12165 }
12166 
12167 void Assembler::pdepq(Register dst, Register src1, Register src2) {
12168   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12169   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12170   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12171   emit_int16((unsigned char)0xF5, (0xC0 | encode));
12172 }
12173 
12174 void Assembler::pextl(Register dst, Register src1, Address src2) {
12175   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12176   InstructionMark im(this);
12177   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12178   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12179   emit_int8((unsigned char)0xF5);
12180   emit_operand(dst, src2, 0);
12181 }
12182 
12183 void Assembler::pdepl(Register dst, Register src1, Address src2) {
12184   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12185   InstructionMark im(this);
12186   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12187   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12188   emit_int8((unsigned char)0xF5);
12189   emit_operand(dst, src2, 0);
12190 }
12191 
12192 void Assembler::pextq(Register dst, Register src1, Address src2) {
12193   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12194   InstructionMark im(this);
12195   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12196   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12197   emit_int8((unsigned char)0xF5);
12198   emit_operand(dst, src2, 0);
12199 }
12200 
12201 void Assembler::pdepq(Register dst, Register src1, Address src2) {
12202   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12203   InstructionMark im(this);
12204   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12205   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12206   emit_int8((unsigned char)0xF5);
12207   emit_operand(dst, src2, 0);
12208 }
12209 
12210 void Assembler::sarxl(Register dst, Register src1, Register src2) {
12211   assert(VM_Version::supports_bmi2(), "");
12212   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12213   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12214   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12215 }
12216 
12217 void Assembler::sarxl(Register dst, Address src1, Register src2) {
12218   assert(VM_Version::supports_bmi2(), "");
12219   InstructionMark im(this);
12220   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12221   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12222   emit_int8((unsigned char)0xF7);
12223   emit_operand(dst, src1, 0);
12224 }
12225 
12226 void Assembler::sarxq(Register dst, Register src1, Register src2) {
12227   assert(VM_Version::supports_bmi2(), "");
12228   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12229   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12230   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12231 }
12232 
12233 void Assembler::sarxq(Register dst, Address src1, Register src2) {
12234   assert(VM_Version::supports_bmi2(), "");
12235   InstructionMark im(this);
12236   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12237   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12238   emit_int8((unsigned char)0xF7);
12239   emit_operand(dst, src1, 0);
12240 }
12241 
12242 void Assembler::shlxl(Register dst, Register src1, Register src2) {
12243   assert(VM_Version::supports_bmi2(), "");
12244   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12245   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12246   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12247 }
12248 
12249 void Assembler::shlxl(Register dst, Address src1, Register src2) {
12250   assert(VM_Version::supports_bmi2(), "");
12251   InstructionMark im(this);
12252   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12253   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12254   emit_int8((unsigned char)0xF7);
12255   emit_operand(dst, src1, 0);
12256 }
12257 
12258 void Assembler::shlxq(Register dst, Register src1, Register src2) {
12259   assert(VM_Version::supports_bmi2(), "");
12260   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12261   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12262   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12263 }
12264 
12265 void Assembler::shlxq(Register dst, Address src1, Register src2) {
12266   assert(VM_Version::supports_bmi2(), "");
12267   InstructionMark im(this);
12268   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12269   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12270   emit_int8((unsigned char)0xF7);
12271   emit_operand(dst, src1, 0);
12272 }
12273 
12274 void Assembler::shrxl(Register dst, Register src1, Register src2) {
12275   assert(VM_Version::supports_bmi2(), "");
12276   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12277   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12278   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12279 }
12280 
12281 void Assembler::shrxl(Register dst, Address src1, Register src2) {
12282   assert(VM_Version::supports_bmi2(), "");
12283   InstructionMark im(this);
12284   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12285   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12286   emit_int8((unsigned char)0xF7);
12287   emit_operand(dst, src1, 0);
12288 }
12289 
12290 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12291   assert(VM_Version::supports_bmi2(), "");
12292   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12293   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12294   emit_int16((unsigned char)0xF7, (0xC0 | encode));
12295 }
12296 
12297 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12298   assert(VM_Version::supports_bmi2(), "");
12299   InstructionMark im(this);
12300   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12301   vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12302   emit_int8((unsigned char)0xF7);
12303   emit_operand(dst, src1, 0);
12304 }
12305 
12306 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12307   assert(VM_Version::supports_avx512vldq(), "");
12308   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12309   attributes.set_is_evex_instruction();
12310   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12311   emit_int16(0x39, (0xC0 | encode));
12312 }
12313 
12314 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12315   assert(VM_Version::supports_avx512vldq(), "");
12316   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12317   attributes.set_is_evex_instruction();
12318   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12319   emit_int16(0x39, (0xC0 | encode));
12320 }
12321 
12322 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12323   assert(VM_Version::supports_avx512vlbw(), "");
12324   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12325   attributes.set_is_evex_instruction();
12326   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12327   emit_int16(0x29, (0xC0 | encode));
12328 }
12329 
12330 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12331   assert(VM_Version::supports_avx512vlbw(), "");
12332   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12333   attributes.set_is_evex_instruction();
12334   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12335   emit_int16(0x29, (0xC0 | encode));
12336 }
12337 
12338 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12339   assert(VM_Version::supports_avx512vldq(), "");
12340   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12341   attributes.set_is_evex_instruction();
12342   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12343   emit_int16(0x38, (0xC0 | encode));
12344 }
12345 
12346 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12347   assert(VM_Version::supports_avx512vldq(), "");
12348   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12349   attributes.set_is_evex_instruction();
12350   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12351   emit_int16(0x38, (0xC0 | encode));
12352 }
12353 
12354 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12355   assert(VM_Version::supports_avx512vlbw(), "");
12356   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12357   attributes.set_is_evex_instruction();
12358   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12359   emit_int16(0x28, (0xC0 | encode));
12360 }
12361 
12362 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12363   assert(VM_Version::supports_avx512vlbw(), "");
12364   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12365   attributes.set_is_evex_instruction();
12366   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12367   emit_int16(0x28, (0xC0 | encode));
12368 }
12369 
12370 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12371   assert(VM_Version::supports_avx512_vbmi2(), "");
12372   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12373   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12374   attributes.set_embedded_opmask_register_specifier(mask);
12375   attributes.set_is_evex_instruction();
12376   if (merge) {
12377     attributes.reset_is_clear_context();
12378   }
12379   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12380   emit_int16((unsigned char)0x63, (0xC0 | encode));
12381 }
12382 
12383 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12384   assert(VM_Version::supports_avx512_vbmi2(), "");
12385   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12386   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12387   attributes.set_embedded_opmask_register_specifier(mask);
12388   attributes.set_is_evex_instruction();
12389   if (merge) {
12390     attributes.reset_is_clear_context();
12391   }
12392   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12393   emit_int16((unsigned char)0x63, (0xC0 | encode));
12394 }
12395 
12396 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12397   assert(VM_Version::supports_evex(), "");
12398   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12400   attributes.set_embedded_opmask_register_specifier(mask);
12401   attributes.set_is_evex_instruction();
12402   if (merge) {
12403     attributes.reset_is_clear_context();
12404   }
12405   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12406   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12407 }
12408 
12409 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12410   assert(VM_Version::supports_evex(), "");
12411   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12412   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12413   attributes.set_embedded_opmask_register_specifier(mask);
12414   attributes.set_is_evex_instruction();
12415   if (merge) {
12416     attributes.reset_is_clear_context();
12417   }
12418   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12419   emit_int16((unsigned char)0x8B, (0xC0 | encode));
12420 }
12421 
12422 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12423   assert(VM_Version::supports_evex(), "");
12424   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12425   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12426   attributes.set_embedded_opmask_register_specifier(mask);
12427   attributes.set_is_evex_instruction();
12428   if (merge) {
12429     attributes.reset_is_clear_context();
12430   }
12431   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12432   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12433 }
12434 
12435 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12436   assert(VM_Version::supports_evex(), "");
12437   assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12438   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12439   attributes.set_embedded_opmask_register_specifier(mask);
12440   attributes.set_is_evex_instruction();
12441   if (merge) {
12442     attributes.reset_is_clear_context();
12443   }
12444   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12445   emit_int16((unsigned char)0x8A, (0xC0 | encode));
12446 }
12447 
12448 #ifndef _LP64
12449 
12450 void Assembler::incl(Register dst) {
12451   // Don't use it directly. Use MacroAssembler::incrementl() instead.
12452   emit_int8(0x40 | dst->encoding());
12453 }
12454 
12455 void Assembler::lea(Register dst, Address src) {
12456   leal(dst, src);
12457 }
12458 
12459 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12460   InstructionMark im(this);
12461   emit_int8((unsigned char)0xC7);
12462   emit_operand(rax, dst, 4);
12463   emit_data((int)imm32, rspec, 0);
12464 }
12465 
12466 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12467   InstructionMark im(this);
12468   int encode = prefix_and_encode(dst->encoding());
12469   emit_int8((0xB8 | encode));
12470   emit_data((int)imm32, rspec, 0);
12471 }
12472 
12473 void Assembler::popa() { // 32bit
12474   emit_int8(0x61);
12475 }
12476 
12477 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12478   InstructionMark im(this);
12479   emit_int8(0x68);
12480   emit_data(imm32, rspec, 0);
12481 }
12482 
12483 void Assembler::pusha() { // 32bit
12484   emit_int8(0x60);
12485 }
12486 
12487 #else // LP64
12488 
12489 // 64bit only pieces of the assembler
12490 
12491 // This should only be used by 64bit instructions that can use rip-relative
12492 // it cannot be used by instructions that want an immediate value.
12493 
12494 // Determine whether an address is always reachable in rip-relative addressing mode
12495 // when accessed from the code cache.
12496 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12497   switch (reloc_type) {
12498     // This should be rip-relative and easily reachable.
12499     case relocInfo::internal_word_type: {
12500       return true;
12501     }
12502     // This should be rip-relative within the code cache and easily
12503     // reachable until we get huge code caches. (At which point
12504     // IC code is going to have issues).
12505     case relocInfo::virtual_call_type:
12506     case relocInfo::opt_virtual_call_type:
12507     case relocInfo::static_call_type:
12508     case relocInfo::static_stub_type: {
12509       return true;
12510     }
12511     case relocInfo::runtime_call_type:
12512     case relocInfo::external_word_type:
12513     case relocInfo::poll_return_type: // these are really external_word but need special
12514     case relocInfo::poll_type: {      // relocs to identify them
12515       return CodeCache::contains(target);
12516     }
12517     default: {
12518       return false;
12519     }
12520   }
12521 }
12522 
12523 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12524 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12525   if (is_always_reachable(target, reloc_type)) {
12526     return true;
12527   }
12528   switch (reloc_type) {
12529     // None will force a 64bit literal to the code stream. Likely a placeholder
12530     // for something that will be patched later and we need to certain it will
12531     // always be reachable.
12532     case relocInfo::none: {
12533       return false;
12534     }
12535     case relocInfo::runtime_call_type:
12536     case relocInfo::external_word_type:
12537     case relocInfo::poll_return_type: // these are really external_word but need special
12538     case relocInfo::poll_type: {      // relocs to identify them
12539       assert(!CodeCache::contains(target), "always reachable");
12540       if (ForceUnreachable) {
12541         return false; // stress the correction code
12542       }
12543       // For external_word_type/runtime_call_type if it is reachable from where we
12544       // are now (possibly a temp buffer) and where we might end up
12545       // anywhere in the code cache then we are always reachable.
12546       // This would have to change if we ever save/restore shared code to be more pessimistic.
12547       // Code buffer has to be allocated in the code cache, so check against
12548       // code cache boundaries cover that case.
12549       //
12550       // In rip-relative addressing mode, an effective address is formed by adding displacement
12551       // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12552       // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12553       // to account for that.
12554       return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12555              Assembler::is_simm32(target - CodeCache::high_bound());
12556     }
12557     default: {
12558       return false;
12559     }
12560   }
12561 }
12562 
12563 bool Assembler::reachable(AddressLiteral adr) {
12564   assert(CodeCache::contains(pc()), "required");
12565   if (adr.is_lval()) {
12566     return false;
12567   }
12568   return is_reachable(adr.target(), adr.reloc());
12569 }
12570 
12571 bool Assembler::always_reachable(AddressLiteral adr) {
12572   assert(CodeCache::contains(pc()), "required");
12573   if (adr.is_lval()) {
12574     return false;
12575   }
12576   return is_always_reachable(adr.target(), adr.reloc());
12577 }
12578 
12579 void Assembler::emit_data64(jlong data,
12580                             relocInfo::relocType rtype,
12581                             int format) {
12582   if (rtype == relocInfo::none) {
12583     emit_int64(data);
12584   } else {
12585     emit_data64(data, Relocation::spec_simple(rtype), format);
12586   }
12587 }
12588 
12589 void Assembler::emit_data64(jlong data,
12590                             RelocationHolder const& rspec,
12591                             int format) {
12592   assert(imm_operand == 0, "default format must be immediate in this file");
12593   assert(imm_operand == format, "must be immediate");
12594   assert(inst_mark() != nullptr, "must be inside InstructionMark");
12595   // Do not use AbstractAssembler::relocate, which is not intended for
12596   // embedded words.  Instead, relocate to the enclosing instruction.
12597   code_section()->relocate(inst_mark(), rspec, format);
12598 #ifdef ASSERT
12599   check_relocation(rspec, format);
12600 #endif
12601   emit_int64(data);
12602 }
12603 
12604 void Assembler::prefix(Register reg) {
12605   if (reg->encoding() >= 8) {
12606     prefix(REX_B);
12607   }
12608 }
12609 
12610 void Assembler::prefix(Register dst, Register src, Prefix p) {
12611   if (src->encoding() >= 8) {
12612     p = (Prefix)(p | REX_B);
12613   }
12614   if (dst->encoding() >= 8) {
12615     p = (Prefix)(p | REX_R);
12616   }
12617   if (p != Prefix_EMPTY) {
12618     // do not generate an empty prefix
12619     prefix(p);
12620   }
12621 }
12622 
12623 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12624   if (adr.base_needs_rex()) {
12625     if (adr.index_needs_rex()) {
12626       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12627     } else {
12628       p = (Prefix)(p | REX_B);
12629     }
12630   } else {
12631     if (adr.index_needs_rex()) {
12632       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12633     }
12634   }
12635   if (dst->encoding() >= 8) {
12636     p = (Prefix)(p | REX_R);
12637   }
12638   if (p != Prefix_EMPTY) {
12639     // do not generate an empty prefix
12640     prefix(p);
12641   }
12642 }
12643 
12644 void Assembler::prefix(Address adr) {
12645   if (adr.base_needs_rex()) {
12646     if (adr.index_needs_rex()) {
12647       prefix(REX_XB);
12648     } else {
12649       prefix(REX_B);
12650     }
12651   } else {
12652     if (adr.index_needs_rex()) {
12653       prefix(REX_X);
12654     }
12655   }
12656 }
12657 
12658 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12659   if (reg->encoding() < 8) {
12660     if (adr.base_needs_rex()) {
12661       if (adr.index_needs_rex()) {
12662         prefix(REX_XB);
12663       } else {
12664         prefix(REX_B);
12665       }
12666     } else {
12667       if (adr.index_needs_rex()) {
12668         prefix(REX_X);
12669       } else if (byteinst && reg->encoding() >= 4) {
12670         prefix(REX);
12671       }
12672     }
12673   } else {
12674     if (adr.base_needs_rex()) {
12675       if (adr.index_needs_rex()) {
12676         prefix(REX_RXB);
12677       } else {
12678         prefix(REX_RB);
12679       }
12680     } else {
12681       if (adr.index_needs_rex()) {
12682         prefix(REX_RX);
12683       } else {
12684         prefix(REX_R);
12685       }
12686     }
12687   }
12688 }
12689 
12690 void Assembler::prefix(Address adr, XMMRegister reg) {
12691   if (reg->encoding() < 8) {
12692     if (adr.base_needs_rex()) {
12693       if (adr.index_needs_rex()) {
12694         prefix(REX_XB);
12695       } else {
12696         prefix(REX_B);
12697       }
12698     } else {
12699       if (adr.index_needs_rex()) {
12700         prefix(REX_X);
12701       }
12702     }
12703   } else {
12704     if (adr.base_needs_rex()) {
12705       if (adr.index_needs_rex()) {
12706         prefix(REX_RXB);
12707       } else {
12708         prefix(REX_RB);
12709       }
12710     } else {
12711       if (adr.index_needs_rex()) {
12712         prefix(REX_RX);
12713       } else {
12714         prefix(REX_R);
12715       }
12716     }
12717   }
12718 }
12719 
12720 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12721   if (reg_enc >= 8) {
12722     prefix(REX_B);
12723     reg_enc -= 8;
12724   } else if (byteinst && reg_enc >= 4) {
12725     prefix(REX);
12726   }
12727   return reg_enc;
12728 }
12729 
12730 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12731   if (dst_enc < 8) {
12732     if (src_enc >= 8) {
12733       prefix(REX_B);
12734       src_enc -= 8;
12735     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12736       prefix(REX);
12737     }
12738   } else {
12739     if (src_enc < 8) {
12740       prefix(REX_R);
12741     } else {
12742       prefix(REX_RB);
12743       src_enc -= 8;
12744     }
12745     dst_enc -= 8;
12746   }
12747   return dst_enc << 3 | src_enc;
12748 }
12749 
12750 int8_t Assembler::get_prefixq(Address adr) {
12751   int8_t prfx = get_prefixq(adr, rax);
12752   assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12753   return prfx;
12754 }
12755 
12756 int8_t Assembler::get_prefixq(Address adr, Register src) {
12757   int8_t prfx = (int8_t)(REX_W +
12758                          ((int)adr.base_needs_rex()) +
12759                          ((int)adr.index_needs_rex() << 1) +
12760                          ((int)(src->encoding() >= 8) << 2));
12761 #ifdef ASSERT
12762   if (src->encoding() < 8) {
12763     if (adr.base_needs_rex()) {
12764       if (adr.index_needs_rex()) {
12765         assert(prfx == REX_WXB, "must be");
12766       } else {
12767         assert(prfx == REX_WB, "must be");
12768       }
12769     } else {
12770       if (adr.index_needs_rex()) {
12771         assert(prfx == REX_WX, "must be");
12772       } else {
12773         assert(prfx == REX_W, "must be");
12774       }
12775     }
12776   } else {
12777     if (adr.base_needs_rex()) {
12778       if (adr.index_needs_rex()) {
12779         assert(prfx == REX_WRXB, "must be");
12780       } else {
12781         assert(prfx == REX_WRB, "must be");
12782       }
12783     } else {
12784       if (adr.index_needs_rex()) {
12785         assert(prfx == REX_WRX, "must be");
12786       } else {
12787         assert(prfx == REX_WR, "must be");
12788       }
12789     }
12790   }
12791 #endif
12792   return prfx;
12793 }
12794 
12795 void Assembler::prefixq(Address adr) {
12796   emit_int8(get_prefixq(adr));
12797 }
12798 
12799 void Assembler::prefixq(Address adr, Register src) {
12800   emit_int8(get_prefixq(adr, src));
12801 }
12802 
12803 void Assembler::prefixq(Address adr, XMMRegister src) {
12804   if (src->encoding() < 8) {
12805     if (adr.base_needs_rex()) {
12806       if (adr.index_needs_rex()) {
12807         prefix(REX_WXB);
12808       } else {
12809         prefix(REX_WB);
12810       }
12811     } else {
12812       if (adr.index_needs_rex()) {
12813         prefix(REX_WX);
12814       } else {
12815         prefix(REX_W);
12816       }
12817     }
12818   } else {
12819     if (adr.base_needs_rex()) {
12820       if (adr.index_needs_rex()) {
12821         prefix(REX_WRXB);
12822       } else {
12823         prefix(REX_WRB);
12824       }
12825     } else {
12826       if (adr.index_needs_rex()) {
12827         prefix(REX_WRX);
12828       } else {
12829         prefix(REX_WR);
12830       }
12831     }
12832   }
12833 }
12834 
12835 int Assembler::prefixq_and_encode(int reg_enc) {
12836   if (reg_enc < 8) {
12837     prefix(REX_W);
12838   } else {
12839     prefix(REX_WB);
12840     reg_enc -= 8;
12841   }
12842   return reg_enc;
12843 }
12844 
12845 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12846   if (dst_enc < 8) {
12847     if (src_enc < 8) {
12848       prefix(REX_W);
12849     } else {
12850       prefix(REX_WB);
12851       src_enc -= 8;
12852     }
12853   } else {
12854     if (src_enc < 8) {
12855       prefix(REX_WR);
12856     } else {
12857       prefix(REX_WRB);
12858       src_enc -= 8;
12859     }
12860     dst_enc -= 8;
12861   }
12862   return dst_enc << 3 | src_enc;
12863 }
12864 
12865 void Assembler::adcq(Register dst, int32_t imm32) {
12866   (void) prefixq_and_encode(dst->encoding());
12867   emit_arith(0x81, 0xD0, dst, imm32);
12868 }
12869 
12870 void Assembler::adcq(Register dst, Address src) {
12871   InstructionMark im(this);
12872   emit_int16(get_prefixq(src, dst), 0x13);
12873   emit_operand(dst, src, 0);
12874 }
12875 
12876 void Assembler::adcq(Register dst, Register src) {
12877   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12878   emit_arith(0x13, 0xC0, dst, src);
12879 }
12880 
12881 void Assembler::addq(Address dst, int32_t imm32) {
12882   InstructionMark im(this);
12883   prefixq(dst);
12884   emit_arith_operand(0x81, rax, dst, imm32);
12885 }
12886 
12887 void Assembler::addq(Address dst, Register src) {
12888   InstructionMark im(this);
12889   emit_int16(get_prefixq(dst, src), 0x01);
12890   emit_operand(src, dst, 0);
12891 }
12892 
12893 void Assembler::addq(Register dst, int32_t imm32) {
12894   (void) prefixq_and_encode(dst->encoding());
12895   emit_arith(0x81, 0xC0, dst, imm32);
12896 }
12897 
12898 void Assembler::addq(Register dst, Address src) {
12899   InstructionMark im(this);
12900   emit_int16(get_prefixq(src, dst), 0x03);
12901   emit_operand(dst, src, 0);
12902 }
12903 
12904 void Assembler::addq(Register dst, Register src) {
12905   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12906   emit_arith(0x03, 0xC0, dst, src);
12907 }
12908 
12909 void Assembler::adcxq(Register dst, Register src) {
12910   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12911   emit_int8(0x66);
12912   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12913   emit_int32(0x0F,
12914              0x38,
12915              (unsigned char)0xF6,
12916              (0xC0 | encode));
12917 }
12918 
12919 void Assembler::adoxq(Register dst, Register src) {
12920   //assert(VM_Version::supports_adx(), "adx instructions not supported");
12921   emit_int8((unsigned char)0xF3);
12922   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12923   emit_int32(0x0F,
12924              0x38,
12925              (unsigned char)0xF6,
12926              (0xC0 | encode));
12927 }
12928 
12929 void Assembler::andq(Address dst, int32_t imm32) {
12930   InstructionMark im(this);
12931   prefixq(dst);
12932   emit_arith_operand(0x81, as_Register(4), dst, imm32);
12933 }
12934 
12935 void Assembler::andq(Register dst, int32_t imm32) {
12936   (void) prefixq_and_encode(dst->encoding());
12937   emit_arith(0x81, 0xE0, dst, imm32);
12938 }
12939 
12940 void Assembler::andq(Register dst, Address src) {
12941   InstructionMark im(this);
12942   emit_int16(get_prefixq(src, dst), 0x23);
12943   emit_operand(dst, src, 0);
12944 }
12945 
12946 void Assembler::andq(Register dst, Register src) {
12947   (void) prefixq_and_encode(dst->encoding(), src->encoding());
12948   emit_arith(0x23, 0xC0, dst, src);
12949 }
12950 
12951 void Assembler::andq(Address dst, Register src) {
12952   InstructionMark im(this);
12953   emit_int16(get_prefixq(dst, src), 0x21);
12954   emit_operand(src, dst, 0);
12955 }
12956 
12957 void Assembler::andnq(Register dst, Register src1, Register src2) {
12958   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12959   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12960   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12961   emit_int16((unsigned char)0xF2, (0xC0 | encode));
12962 }
12963 
12964 void Assembler::andnq(Register dst, Register src1, Address src2) {
12965   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12966   InstructionMark im(this);
12967   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12968   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12969   emit_int8((unsigned char)0xF2);
12970   emit_operand(dst, src2, 0);
12971 }
12972 
12973 void Assembler::bsfq(Register dst, Register src) {
12974   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12975   emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12976 }
12977 
12978 void Assembler::bsrq(Register dst, Register src) {
12979   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12980   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12981 }
12982 
12983 void Assembler::bswapq(Register reg) {
12984   int encode = prefixq_and_encode(reg->encoding());
12985   emit_int16(0x0F, (0xC8 | encode));
12986 }
12987 
12988 void Assembler::blsiq(Register dst, Register src) {
12989   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12990   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12991   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12992   emit_int16((unsigned char)0xF3, (0xC0 | encode));
12993 }
12994 
12995 void Assembler::blsiq(Register dst, Address src) {
12996   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12997   InstructionMark im(this);
12998   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12999   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
13000   emit_int8((unsigned char)0xF3);
13001   emit_operand(rbx, src, 0);
13002 }
13003 
13004 void Assembler::blsmskq(Register dst, Register src) {
13005   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
13006   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13007   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
13008   emit_int16((unsigned char)0xF3, (0xC0 | encode));
13009 }
13010 
13011 void Assembler::blsmskq(Register dst, Address src) {
13012   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
13013   InstructionMark im(this);
13014   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13015   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
13016   emit_int8((unsigned char)0xF3);
13017   emit_operand(rdx, src, 0);
13018 }
13019 
13020 void Assembler::blsrq(Register dst, Register src) {
13021   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
13022   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13023   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
13024   emit_int16((unsigned char)0xF3, (0xC0 | encode));
13025 }
13026 
13027 void Assembler::blsrq(Register dst, Address src) {
13028   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
13029   InstructionMark im(this);
13030   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13031   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
13032   emit_int8((unsigned char)0xF3);
13033   emit_operand(rcx, src, 0);
13034 }
13035 
13036 void Assembler::cdqq() {
13037   emit_int16(REX_W, (unsigned char)0x99);
13038 }
13039 
13040 void Assembler::clflush(Address adr) {
13041   assert(VM_Version::supports_clflush(), "should do");
13042   prefix(adr);
13043   emit_int16(0x0F, (unsigned char)0xAE);
13044   emit_operand(rdi, adr, 0);
13045 }
13046 
13047 void Assembler::clflushopt(Address adr) {
13048   assert(VM_Version::supports_clflushopt(), "should do!");
13049   // adr should be base reg only with no index or offset
13050   assert(adr.index() == noreg, "index should be noreg");
13051   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
13052   assert(adr.disp() == 0, "displacement should be 0");
13053   // instruction prefix is 0x66
13054   emit_int8(0x66);
13055   prefix(adr);
13056   // opcode family is 0x0F 0xAE
13057   emit_int16(0x0F, (unsigned char)0xAE);
13058   // extended opcode byte is 7 == rdi
13059   emit_operand(rdi, adr, 0);
13060 }
13061 
13062 void Assembler::clwb(Address adr) {
13063   assert(VM_Version::supports_clwb(), "should do!");
13064   // adr should be base reg only with no index or offset
13065   assert(adr.index() == noreg, "index should be noreg");
13066   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
13067   assert(adr.disp() == 0, "displacement should be 0");
13068   // instruction prefix is 0x66
13069   emit_int8(0x66);
13070   prefix(adr);
13071   // opcode family is 0x0f 0xAE
13072   emit_int16(0x0F, (unsigned char)0xAE);
13073   // extended opcode byte is 6 == rsi
13074   emit_operand(rsi, adr, 0);
13075 }
13076 
13077 void Assembler::cmovq(Condition cc, Register dst, Register src) {
13078   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13079   emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
13080 }
13081 
13082 void Assembler::cmovq(Condition cc, Register dst, Address src) {
13083   InstructionMark im(this);
13084   emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
13085   emit_operand(dst, src, 0);
13086 }
13087 
13088 void Assembler::cmpq(Address dst, int32_t imm32) {
13089   InstructionMark im(this);
13090   prefixq(dst);
13091   emit_arith_operand(0x81, as_Register(7), dst, imm32);
13092 }
13093 
13094 void Assembler::cmpq(Register dst, int32_t imm32) {
13095   (void) prefixq_and_encode(dst->encoding());
13096   emit_arith(0x81, 0xF8, dst, imm32);
13097 }
13098 
13099 void Assembler::cmpq(Address dst, Register src) {
13100   InstructionMark im(this);
13101   emit_int16(get_prefixq(dst, src), 0x39);
13102   emit_operand(src, dst, 0);
13103 }
13104 
13105 void Assembler::cmpq(Register dst, Register src) {
13106   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13107   emit_arith(0x3B, 0xC0, dst, src);
13108 }
13109 
13110 void Assembler::cmpq(Register dst, Address src) {
13111   InstructionMark im(this);
13112   emit_int16(get_prefixq(src, dst), 0x3B);
13113   emit_operand(dst, src, 0);
13114 }
13115 
13116 void Assembler::cmpxchgq(Register reg, Address adr) {
13117   InstructionMark im(this);
13118   emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
13119   emit_operand(reg, adr, 0);
13120 }
13121 
13122 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
13123   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13124   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13125   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13126   emit_int16(0x2A, (0xC0 | encode));
13127 }
13128 
13129 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
13130   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13131   InstructionMark im(this);
13132   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13133   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13134   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13135   emit_int8(0x2A);
13136   emit_operand(dst, src, 0);
13137 }
13138 
13139 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
13140   NOT_LP64(assert(VM_Version::supports_sse(), ""));
13141   InstructionMark im(this);
13142   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13143   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13144   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13145   emit_int8(0x2A);
13146   emit_operand(dst, src, 0);
13147 }
13148 
13149 void Assembler::cvttsd2siq(Register dst, Address src) {
13150   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13151   // F2 REX.W 0F 2C /r
13152   // CVTTSD2SI r64, xmm1/m64
13153   InstructionMark im(this);
13154   emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
13155   emit_operand(dst, src, 0);
13156 }
13157 
13158 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
13159   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13160   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13161   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13162   emit_int16(0x2C, (0xC0 | encode));
13163 }
13164 
13165 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
13166   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13167   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13168   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13169   emit_int16(0x2D, (0xC0 | encode));
13170 }
13171 
13172 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
13173   NOT_LP64(assert(VM_Version::supports_sse(), ""));
13174   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13175   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13176   emit_int16(0x2C, (0xC0 | encode));
13177 }
13178 
13179 void Assembler::decl(Register dst) {
13180   // Don't use it directly. Use MacroAssembler::decrementl() instead.
13181   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
13182   int encode = prefix_and_encode(dst->encoding());
13183   emit_int16((unsigned char)0xFF, (0xC8 | encode));
13184 }
13185 
13186 void Assembler::decq(Register dst) {
13187   // Don't use it directly. Use MacroAssembler::decrementq() instead.
13188   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13189   int encode = prefixq_and_encode(dst->encoding());
13190   emit_int16((unsigned char)0xFF, 0xC8 | encode);
13191 }
13192 
13193 void Assembler::decq(Address dst) {
13194   // Don't use it directly. Use MacroAssembler::decrementq() instead.
13195   InstructionMark im(this);
13196   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13197   emit_operand(rcx, dst, 0);
13198 }
13199 
13200 void Assembler::fxrstor(Address src) {
13201   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13202   emit_operand(as_Register(1), src, 0);
13203 }
13204 
13205 void Assembler::xrstor(Address src) {
13206   emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13207   emit_operand(as_Register(5), src, 0);
13208 }
13209 
13210 void Assembler::fxsave(Address dst) {
13211   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13212   emit_operand(as_Register(0), dst, 0);
13213 }
13214 
13215 void Assembler::xsave(Address dst) {
13216   emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13217   emit_operand(as_Register(4), dst, 0);
13218 }
13219 
13220 void Assembler::idivq(Register src) {
13221   int encode = prefixq_and_encode(src->encoding());
13222   emit_int16((unsigned char)0xF7, (0xF8 | encode));
13223 }
13224 
13225 void Assembler::divq(Register src) {
13226   int encode = prefixq_and_encode(src->encoding());
13227   emit_int16((unsigned char)0xF7, (0xF0 | encode));
13228 }
13229 
13230 void Assembler::imulq(Register dst, Register src) {
13231   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13232   emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
13233 }
13234 
13235 void Assembler::imulq(Register src) {
13236   int encode = prefixq_and_encode(src->encoding());
13237   emit_int16((unsigned char)0xF7, (0xE8 | encode));
13238 }
13239 
13240 void Assembler::imulq(Register dst, Address src, int32_t value) {
13241   InstructionMark im(this);
13242   prefixq(src, dst);
13243   if (is8bit(value)) {
13244     emit_int8((unsigned char)0x6B);
13245     emit_operand(dst, src, 1);
13246     emit_int8(value);
13247   } else {
13248     emit_int8((unsigned char)0x69);
13249     emit_operand(dst, src, 4);
13250     emit_int32(value);
13251   }
13252 }
13253 
13254 void Assembler::imulq(Register dst, Register src, int value) {
13255   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13256   if (is8bit(value)) {
13257     emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
13258   } else {
13259     emit_int16(0x69, (0xC0 | encode));
13260     emit_int32(value);
13261   }
13262 }
13263 
13264 void Assembler::imulq(Register dst, Address src) {
13265   InstructionMark im(this);
13266   emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
13267   emit_operand(dst, src, 0);
13268 }
13269 
13270 void Assembler::incl(Register dst) {
13271   // Don't use it directly. Use MacroAssembler::incrementl() instead.
13272   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13273   int encode = prefix_and_encode(dst->encoding());
13274   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13275 }
13276 
13277 void Assembler::incq(Register dst) {
13278   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13279   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13280   int encode = prefixq_and_encode(dst->encoding());
13281   emit_int16((unsigned char)0xFF, (0xC0 | encode));
13282 }
13283 
13284 void Assembler::incq(Address dst) {
13285   // Don't use it directly. Use MacroAssembler::incrementq() instead.
13286   InstructionMark im(this);
13287   emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13288   emit_operand(rax, dst, 0);
13289 }
13290 
13291 void Assembler::lea(Register dst, Address src) {
13292   leaq(dst, src);
13293 }
13294 
13295 void Assembler::leaq(Register dst, Address src) {
13296   InstructionMark im(this);
13297   emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13298   emit_operand(dst, src, 0);
13299 }
13300 
13301 void Assembler::mov64(Register dst, int64_t imm64) {
13302   InstructionMark im(this);
13303   int encode = prefixq_and_encode(dst->encoding());
13304   emit_int8(0xB8 | encode);
13305   emit_int64(imm64);
13306 }
13307 
13308 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13309   InstructionMark im(this);
13310   int encode = prefixq_and_encode(dst->encoding());
13311   emit_int8(0xB8 | encode);
13312   emit_data64(imm64, rtype, format);
13313 }
13314 
13315 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13316   InstructionMark im(this);
13317   int encode = prefixq_and_encode(dst->encoding());
13318   emit_int8(0xB8 | encode);
13319   emit_data64(imm64, rspec);
13320 }
13321 
13322 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13323   InstructionMark im(this);
13324   int encode = prefix_and_encode(dst->encoding());
13325   emit_int8(0xB8 | encode);
13326   emit_data((int)imm32, rspec, narrow_oop_operand);
13327 }
13328 
13329 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
13330   InstructionMark im(this);
13331   prefix(dst);
13332   emit_int8((unsigned char)0xC7);
13333   emit_operand(rax, dst, 4);
13334   emit_data((int)imm32, rspec, narrow_oop_operand);
13335 }
13336 
13337 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13338   InstructionMark im(this);
13339   int encode = prefix_and_encode(src1->encoding());
13340   emit_int16((unsigned char)0x81, (0xF8 | encode));
13341   emit_data((int)imm32, rspec, narrow_oop_operand);
13342 }
13343 
13344 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13345   InstructionMark im(this);
13346   prefix(src1);
13347   emit_int8((unsigned char)0x81);
13348   emit_operand(rax, src1, 4);
13349   emit_data((int)imm32, rspec, narrow_oop_operand);
13350 }
13351 
13352 void Assembler::lzcntq(Register dst, Register src) {
13353   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13354   emit_int8((unsigned char)0xF3);
13355   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13356   emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13357 }
13358 
13359 void Assembler::lzcntq(Register dst, Address src) {
13360   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13361   InstructionMark im(this);
13362   emit_int8((unsigned char)0xF3);
13363   prefixq(src, dst);
13364   emit_int16(0x0F, (unsigned char)0xBD);
13365   emit_operand(dst, src, 0);
13366 }
13367 
13368 void Assembler::movdq(XMMRegister dst, Register src) {
13369   // table D-1 says MMX/SSE2
13370   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13371   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13372   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13373   emit_int16(0x6E, (0xC0 | encode));
13374 }
13375 
13376 void Assembler::movdq(Register dst, XMMRegister src) {
13377   // table D-1 says MMX/SSE2
13378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13379   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13380   // swap src/dst to get correct prefix
13381   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13382   emit_int16(0x7E,
13383              (0xC0 | encode));
13384 }
13385 
13386 void Assembler::movq(Register dst, Register src) {
13387   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13388   emit_int16((unsigned char)0x8B,
13389              (0xC0 | encode));
13390 }
13391 
13392 void Assembler::movq(Register dst, Address src) {
13393   InstructionMark im(this);
13394   emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13395   emit_operand(dst, src, 0);
13396 }
13397 
13398 void Assembler::movq(Address dst, Register src) {
13399   InstructionMark im(this);
13400   emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13401   emit_operand(src, dst, 0);
13402 }
13403 
13404 void Assembler::movq(Address dst, int32_t imm32) {
13405   InstructionMark im(this);
13406   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13407   emit_operand(as_Register(0), dst, 4);
13408   emit_int32(imm32);
13409 }
13410 
13411 void Assembler::movq(Register dst, int32_t imm32) {
13412   int encode = prefixq_and_encode(dst->encoding());
13413   emit_int16((unsigned char)0xC7, (0xC0 | encode));
13414   emit_int32(imm32);
13415 }
13416 
13417 void Assembler::movsbq(Register dst, Address src) {
13418   InstructionMark im(this);
13419   emit_int24(get_prefixq(src, dst),
13420              0x0F,
13421              (unsigned char)0xBE);
13422   emit_operand(dst, src, 0);
13423 }
13424 
13425 void Assembler::movsbq(Register dst, Register src) {
13426   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13427   emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13428 }
13429 
13430 void Assembler::movslq(Register dst, int32_t imm32) {
13431   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
13432   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
13433   // as a result we shouldn't use until tested at runtime...
13434   ShouldNotReachHere();
13435   InstructionMark im(this);
13436   int encode = prefixq_and_encode(dst->encoding());
13437   emit_int8(0xC7 | encode);
13438   emit_int32(imm32);
13439 }
13440 
13441 void Assembler::movslq(Address dst, int32_t imm32) {
13442   assert(is_simm32(imm32), "lost bits");
13443   InstructionMark im(this);
13444   emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13445   emit_operand(rax, dst, 4);
13446   emit_int32(imm32);
13447 }
13448 
13449 void Assembler::movslq(Register dst, Address src) {
13450   InstructionMark im(this);
13451   emit_int16(get_prefixq(src, dst), 0x63);
13452   emit_operand(dst, src, 0);
13453 }
13454 
13455 void Assembler::movslq(Register dst, Register src) {
13456   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13457   emit_int16(0x63, (0xC0 | encode));
13458 }
13459 
13460 void Assembler::movswq(Register dst, Address src) {
13461   InstructionMark im(this);
13462   emit_int24(get_prefixq(src, dst),
13463              0x0F,
13464              (unsigned char)0xBF);
13465   emit_operand(dst, src, 0);
13466 }
13467 
13468 void Assembler::movswq(Register dst, Register src) {
13469   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13470   emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13471 }
13472 
13473 void Assembler::movzbq(Register dst, Address src) {
13474   InstructionMark im(this);
13475   emit_int24(get_prefixq(src, dst),
13476              0x0F,
13477              (unsigned char)0xB6);
13478   emit_operand(dst, src, 0);
13479 }
13480 
13481 void Assembler::movzbq(Register dst, Register src) {
13482   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13483   emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13484 }
13485 
13486 void Assembler::movzwq(Register dst, Address src) {
13487   InstructionMark im(this);
13488   emit_int24(get_prefixq(src, dst),
13489              0x0F,
13490              (unsigned char)0xB7);
13491   emit_operand(dst, src, 0);
13492 }
13493 
13494 void Assembler::movzwq(Register dst, Register src) {
13495   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13496   emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13497 }
13498 
13499 void Assembler::mulq(Address src) {
13500   InstructionMark im(this);
13501   emit_int16(get_prefixq(src), (unsigned char)0xF7);
13502   emit_operand(rsp, src, 0);
13503 }
13504 
13505 void Assembler::mulq(Register src) {
13506   int encode = prefixq_and_encode(src->encoding());
13507   emit_int16((unsigned char)0xF7, (0xE0 | encode));
13508 }
13509 
13510 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13511   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13512   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13513   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13514   emit_int16((unsigned char)0xF6, (0xC0 | encode));
13515 }
13516 
13517 void Assembler::negq(Register dst) {
13518   int encode = prefixq_and_encode(dst->encoding());
13519   emit_int16((unsigned char)0xF7, (0xD8 | encode));
13520 }
13521 
13522 void Assembler::negq(Address dst) {
13523   InstructionMark im(this);
13524   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13525   emit_operand(as_Register(3), dst, 0);
13526 }
13527 
13528 void Assembler::notq(Register dst) {
13529   int encode = prefixq_and_encode(dst->encoding());
13530   emit_int16((unsigned char)0xF7, (0xD0 | encode));
13531 }
13532 
13533 void Assembler::btsq(Address dst, int imm8) {
13534   assert(isByte(imm8), "not a byte");
13535   InstructionMark im(this);
13536   emit_int24(get_prefixq(dst),
13537              0x0F,
13538              (unsigned char)0xBA);
13539   emit_operand(rbp /* 5 */, dst, 1);
13540   emit_int8(imm8);
13541 }
13542 
13543 void Assembler::btrq(Address dst, int imm8) {
13544   assert(isByte(imm8), "not a byte");
13545   InstructionMark im(this);
13546   emit_int24(get_prefixq(dst),
13547              0x0F,
13548              (unsigned char)0xBA);
13549   emit_operand(rsi /* 6 */, dst, 1);
13550   emit_int8(imm8);
13551 }
13552 
13553 void Assembler::orq(Address dst, int32_t imm32) {
13554   InstructionMark im(this);
13555   prefixq(dst);
13556   emit_arith_operand(0x81, as_Register(1), dst, imm32);
13557 }
13558 
13559 void Assembler::orq(Address dst, Register src) {
13560   InstructionMark im(this);
13561   emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13562   emit_operand(src, dst, 0);
13563 }
13564 
13565 void Assembler::orq(Register dst, int32_t imm32) {
13566   (void) prefixq_and_encode(dst->encoding());
13567   emit_arith(0x81, 0xC8, dst, imm32);
13568 }
13569 
13570 void Assembler::orq_imm32(Register dst, int32_t imm32) {
13571   (void) prefixq_and_encode(dst->encoding());
13572   emit_arith_imm32(0x81, 0xC8, dst, imm32);
13573 }
13574 
13575 void Assembler::orq(Register dst, Address src) {
13576   InstructionMark im(this);
13577   emit_int16(get_prefixq(src, dst), 0x0B);
13578   emit_operand(dst, src, 0);
13579 }
13580 
13581 void Assembler::orq(Register dst, Register src) {
13582   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13583   emit_arith(0x0B, 0xC0, dst, src);
13584 }
13585 
13586 void Assembler::popcntq(Register dst, Address src) {
13587   assert(VM_Version::supports_popcnt(), "must support");
13588   InstructionMark im(this);
13589   emit_int32((unsigned char)0xF3,
13590              get_prefixq(src, dst),
13591              0x0F,
13592              (unsigned char)0xB8);
13593   emit_operand(dst, src, 0);
13594 }
13595 
13596 void Assembler::popcntq(Register dst, Register src) {
13597   assert(VM_Version::supports_popcnt(), "must support");
13598   emit_int8((unsigned char)0xF3);
13599   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13600   emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13601 }
13602 
13603 void Assembler::popq(Address dst) {
13604   InstructionMark im(this);
13605   emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13606   emit_operand(rax, dst, 0);
13607 }
13608 
13609 void Assembler::popq(Register dst) {
13610   emit_int8((unsigned char)0x58 | dst->encoding());
13611 }
13612 
13613 // Precomputable: popa, pusha, vzeroupper
13614 
13615 // The result of these routines are invariant from one invocation to another
13616 // invocation for the duration of a run. Caching the result on bootstrap
13617 // and copying it out on subsequent invocations can thus be beneficial
13618 static bool     precomputed = false;
13619 
13620 static u_char* popa_code  = nullptr;
13621 static int     popa_len   = 0;
13622 
13623 static u_char* pusha_code = nullptr;
13624 static int     pusha_len  = 0;
13625 
13626 static u_char* vzup_code  = nullptr;
13627 static int     vzup_len   = 0;
13628 
13629 void Assembler::precompute_instructions() {
13630   assert(!Universe::is_fully_initialized(), "must still be single threaded");
13631   guarantee(!precomputed, "only once");
13632   precomputed = true;
13633   ResourceMark rm;
13634 
13635   // Make a temporary buffer big enough for the routines we're capturing
13636   int size = 256;
13637   char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13638   CodeBuffer buffer((address)tmp_code, size);
13639   MacroAssembler masm(&buffer);
13640 
13641   address begin_popa  = masm.code_section()->end();
13642   masm.popa_uncached();
13643   address end_popa    = masm.code_section()->end();
13644   masm.pusha_uncached();
13645   address end_pusha   = masm.code_section()->end();
13646   masm.vzeroupper_uncached();
13647   address end_vzup    = masm.code_section()->end();
13648 
13649   // Save the instructions to permanent buffers.
13650   popa_len = (int)(end_popa - begin_popa);
13651   popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13652   memcpy(popa_code, begin_popa, popa_len);
13653 
13654   pusha_len = (int)(end_pusha - end_popa);
13655   pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13656   memcpy(pusha_code, end_popa, pusha_len);
13657 
13658   vzup_len = (int)(end_vzup - end_pusha);
13659   if (vzup_len > 0) {
13660     vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13661     memcpy(vzup_code, end_pusha, vzup_len);
13662   } else {
13663     vzup_code = pusha_code; // dummy
13664   }
13665 
13666   assert(masm.code()->total_oop_size() == 0 &&
13667          masm.code()->total_metadata_size() == 0 &&
13668          masm.code()->total_relocation_size() == 0,
13669          "pre-computed code can't reference oops, metadata or contain relocations");
13670 }
13671 
13672 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13673   assert(src != nullptr, "code to copy must have been pre-computed");
13674   assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13675   address end = code_section->end();
13676   memcpy(end, src, src_len);
13677   code_section->set_end(end + src_len);
13678 }
13679 
13680 void Assembler::popa() { // 64bit
13681   emit_copy(code_section(), popa_code, popa_len);
13682 }
13683 
13684 void Assembler::popa_uncached() { // 64bit
13685   movq(r15, Address(rsp, 0));
13686   movq(r14, Address(rsp, wordSize));
13687   movq(r13, Address(rsp, 2 * wordSize));
13688   movq(r12, Address(rsp, 3 * wordSize));
13689   movq(r11, Address(rsp, 4 * wordSize));
13690   movq(r10, Address(rsp, 5 * wordSize));
13691   movq(r9,  Address(rsp, 6 * wordSize));
13692   movq(r8,  Address(rsp, 7 * wordSize));
13693   movq(rdi, Address(rsp, 8 * wordSize));
13694   movq(rsi, Address(rsp, 9 * wordSize));
13695   movq(rbp, Address(rsp, 10 * wordSize));
13696   // Skip rsp as it is restored automatically to the value
13697   // before the corresponding pusha when popa is done.
13698   movq(rbx, Address(rsp, 12 * wordSize));
13699   movq(rdx, Address(rsp, 13 * wordSize));
13700   movq(rcx, Address(rsp, 14 * wordSize));
13701   movq(rax, Address(rsp, 15 * wordSize));
13702 
13703   addq(rsp, 16 * wordSize);
13704 }
13705 
13706 // Does not actually store the value of rsp on the stack.
13707 // The slot for rsp just contains an arbitrary value.
13708 void Assembler::pusha() { // 64bit
13709   emit_copy(code_section(), pusha_code, pusha_len);
13710 }
13711 
13712 // Does not actually store the value of rsp on the stack.
13713 // The slot for rsp just contains an arbitrary value.
13714 void Assembler::pusha_uncached() { // 64bit
13715   subq(rsp, 16 * wordSize);
13716 
13717   movq(Address(rsp, 15 * wordSize), rax);
13718   movq(Address(rsp, 14 * wordSize), rcx);
13719   movq(Address(rsp, 13 * wordSize), rdx);
13720   movq(Address(rsp, 12 * wordSize), rbx);
13721   // Skip rsp as the value is normally not used. There are a few places where
13722   // the original value of rsp needs to be known but that can be computed
13723   // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13724   movq(Address(rsp, 10 * wordSize), rbp);
13725   movq(Address(rsp, 9 * wordSize), rsi);
13726   movq(Address(rsp, 8 * wordSize), rdi);
13727   movq(Address(rsp, 7 * wordSize), r8);
13728   movq(Address(rsp, 6 * wordSize), r9);
13729   movq(Address(rsp, 5 * wordSize), r10);
13730   movq(Address(rsp, 4 * wordSize), r11);
13731   movq(Address(rsp, 3 * wordSize), r12);
13732   movq(Address(rsp, 2 * wordSize), r13);
13733   movq(Address(rsp, wordSize), r14);
13734   movq(Address(rsp, 0), r15);
13735 }
13736 
13737 void Assembler::vzeroupper() {
13738   emit_copy(code_section(), vzup_code, vzup_len);
13739 }
13740 
13741 void Assembler::vzeroall() {
13742   assert(VM_Version::supports_avx(), "requires AVX");
13743   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13744   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
13745   emit_int8(0x77);
13746 }
13747 
13748 void Assembler::pushq(Address src) {
13749   InstructionMark im(this);
13750   emit_int16(get_prefixq(src), (unsigned char)0xFF);
13751   emit_operand(rsi, src, 0);
13752 }
13753 
13754 void Assembler::rclq(Register dst, int imm8) {
13755   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13756   int encode = prefixq_and_encode(dst->encoding());
13757   if (imm8 == 1) {
13758     emit_int16((unsigned char)0xD1, (0xD0 | encode));
13759   } else {
13760     emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13761   }
13762 }
13763 
13764 void Assembler::rcrq(Register dst, int imm8) {
13765   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13766   int encode = prefixq_and_encode(dst->encoding());
13767   if (imm8 == 1) {
13768     emit_int16((unsigned char)0xD1, (0xD8 | encode));
13769   } else {
13770     emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13771   }
13772 }
13773 
13774 void Assembler::rorxl(Register dst, Register src, int imm8) {
13775   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13776   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13777   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13778   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13779 }
13780 
13781 void Assembler::rorxl(Register dst, Address src, int imm8) {
13782   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13783   InstructionMark im(this);
13784   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13785   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13786   emit_int8((unsigned char)0xF0);
13787   emit_operand(dst, src, 1);
13788   emit_int8(imm8);
13789 }
13790 
13791 void Assembler::rorxq(Register dst, Register src, int imm8) {
13792   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13793   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13794   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13795   emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13796 }
13797 
13798 void Assembler::rorxq(Register dst, Address src, int imm8) {
13799   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13800   InstructionMark im(this);
13801   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13802   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13803   emit_int8((unsigned char)0xF0);
13804   emit_operand(dst, src, 1);
13805   emit_int8(imm8);
13806 }
13807 
13808 #ifdef _LP64
13809 void Assembler::salq(Address dst, int imm8) {
13810   InstructionMark im(this);
13811   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13812   if (imm8 == 1) {
13813     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13814     emit_operand(as_Register(4), dst, 0);
13815   }
13816   else {
13817     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13818     emit_operand(as_Register(4), dst, 1);
13819     emit_int8(imm8);
13820   }
13821 }
13822 
13823 void Assembler::salq(Address dst) {
13824   InstructionMark im(this);
13825   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13826   emit_operand(as_Register(4), dst, 0);
13827 }
13828 
13829 void Assembler::salq(Register dst, int imm8) {
13830   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13831   int encode = prefixq_and_encode(dst->encoding());
13832   if (imm8 == 1) {
13833     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13834   } else {
13835     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13836   }
13837 }
13838 
13839 void Assembler::salq(Register dst) {
13840   int encode = prefixq_and_encode(dst->encoding());
13841   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13842 }
13843 
13844 void Assembler::sarq(Address dst, int imm8) {
13845   InstructionMark im(this);
13846   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13847   if (imm8 == 1) {
13848     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13849     emit_operand(as_Register(7), dst, 0);
13850   }
13851   else {
13852     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13853     emit_operand(as_Register(7), dst, 1);
13854     emit_int8(imm8);
13855   }
13856 }
13857 
13858 void Assembler::sarq(Address dst) {
13859   InstructionMark im(this);
13860   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13861   emit_operand(as_Register(7), dst, 0);
13862 }
13863 
13864 void Assembler::sarq(Register dst, int imm8) {
13865   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13866   int encode = prefixq_and_encode(dst->encoding());
13867   if (imm8 == 1) {
13868     emit_int16((unsigned char)0xD1, (0xF8 | encode));
13869   } else {
13870     emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13871   }
13872 }
13873 
13874 void Assembler::sarq(Register dst) {
13875   int encode = prefixq_and_encode(dst->encoding());
13876   emit_int16((unsigned char)0xD3, (0xF8 | encode));
13877 }
13878 #endif
13879 
13880 void Assembler::sbbq(Address dst, int32_t imm32) {
13881   InstructionMark im(this);
13882   prefixq(dst);
13883   emit_arith_operand(0x81, rbx, dst, imm32);
13884 }
13885 
13886 void Assembler::sbbq(Register dst, int32_t imm32) {
13887   (void) prefixq_and_encode(dst->encoding());
13888   emit_arith(0x81, 0xD8, dst, imm32);
13889 }
13890 
13891 void Assembler::sbbq(Register dst, Address src) {
13892   InstructionMark im(this);
13893   emit_int16(get_prefixq(src, dst), 0x1B);
13894   emit_operand(dst, src, 0);
13895 }
13896 
13897 void Assembler::sbbq(Register dst, Register src) {
13898   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13899   emit_arith(0x1B, 0xC0, dst, src);
13900 }
13901 
13902 void Assembler::shlq(Register dst, int imm8) {
13903   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13904   int encode = prefixq_and_encode(dst->encoding());
13905   if (imm8 == 1) {
13906     emit_int16((unsigned char)0xD1, (0xE0 | encode));
13907   } else {
13908     emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13909   }
13910 }
13911 
13912 void Assembler::shlq(Register dst) {
13913   int encode = prefixq_and_encode(dst->encoding());
13914   emit_int16((unsigned char)0xD3, (0xE0 | encode));
13915 }
13916 
13917 void Assembler::shrq(Register dst, int imm8) {
13918   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13919   int encode = prefixq_and_encode(dst->encoding());
13920   if (imm8 == 1) {
13921     emit_int16((unsigned char)0xD1, (0xE8 | encode));
13922   }
13923   else {
13924     emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13925   }
13926 }
13927 
13928 void Assembler::shrq(Register dst) {
13929   int encode = prefixq_and_encode(dst->encoding());
13930   emit_int16((unsigned char)0xD3, 0xE8 | encode);
13931 }
13932 
13933 void Assembler::shrq(Address dst) {
13934   InstructionMark im(this);
13935   emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13936   emit_operand(as_Register(5), dst, 0);
13937 }
13938 
13939 void Assembler::shrq(Address dst, int imm8) {
13940   InstructionMark im(this);
13941   assert(isShiftCount(imm8 >> 1), "illegal shift count");
13942   if (imm8 == 1) {
13943     emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13944     emit_operand(as_Register(5), dst, 0);
13945   }
13946   else {
13947     emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13948     emit_operand(as_Register(5), dst, 1);
13949     emit_int8(imm8);
13950   }
13951 }
13952 
13953 void Assembler::subq(Address dst, int32_t imm32) {
13954   InstructionMark im(this);
13955   prefixq(dst);
13956   emit_arith_operand(0x81, rbp, dst, imm32);
13957 }
13958 
13959 void Assembler::subq(Address dst, Register src) {
13960   InstructionMark im(this);
13961   emit_int16(get_prefixq(dst, src), 0x29);
13962   emit_operand(src, dst, 0);
13963 }
13964 
13965 void Assembler::subq(Register dst, int32_t imm32) {
13966   (void) prefixq_and_encode(dst->encoding());
13967   emit_arith(0x81, 0xE8, dst, imm32);
13968 }
13969 
13970 // Force generation of a 4 byte immediate value even if it fits into 8bit
13971 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13972   (void) prefixq_and_encode(dst->encoding());
13973   emit_arith_imm32(0x81, 0xE8, dst, imm32);
13974 }
13975 
13976 void Assembler::subq(Register dst, Address src) {
13977   InstructionMark im(this);
13978   emit_int16(get_prefixq(src, dst), 0x2B);
13979   emit_operand(dst, src, 0);
13980 }
13981 
13982 void Assembler::subq(Register dst, Register src) {
13983   (void) prefixq_and_encode(dst->encoding(), src->encoding());
13984   emit_arith(0x2B, 0xC0, dst, src);
13985 }
13986 
13987 void Assembler::testq(Address dst, int32_t imm32) {
13988   InstructionMark im(this);
13989   emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13990   emit_operand(as_Register(0), dst, 4);
13991   emit_int32(imm32);
13992 }
13993 
13994 void Assembler::testq(Register dst, int32_t imm32) {
13995   // not using emit_arith because test
13996   // doesn't support sign-extension of
13997   // 8bit operands
13998   if (dst == rax) {
13999     prefix(REX_W);
14000     emit_int8((unsigned char)0xA9);
14001     emit_int32(imm32);
14002   } else {
14003     int encode = dst->encoding();
14004     encode = prefixq_and_encode(encode);
14005     emit_int16((unsigned char)0xF7, (0xC0 | encode));
14006     emit_int32(imm32);
14007   }
14008 }
14009 
14010 void Assembler::testq(Register dst, Register src) {
14011   (void) prefixq_and_encode(dst->encoding(), src->encoding());
14012   emit_arith(0x85, 0xC0, dst, src);
14013 }
14014 
14015 void Assembler::testq(Register dst, Address src) {
14016   InstructionMark im(this);
14017   emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
14018   emit_operand(dst, src, 0);
14019 }
14020 
14021 void Assembler::xaddq(Address dst, Register src) {
14022   InstructionMark im(this);
14023   emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
14024   emit_operand(src, dst, 0);
14025 }
14026 
14027 void Assembler::xchgq(Register dst, Address src) {
14028   InstructionMark im(this);
14029   emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
14030   emit_operand(dst, src, 0);
14031 }
14032 
14033 void Assembler::xchgq(Register dst, Register src) {
14034   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
14035   emit_int16((unsigned char)0x87, (0xc0 | encode));
14036 }
14037 
14038 void Assembler::xorq(Register dst, Register src) {
14039   (void) prefixq_and_encode(dst->encoding(), src->encoding());
14040   emit_arith(0x33, 0xC0, dst, src);
14041 }
14042 
14043 void Assembler::xorq(Register dst, Address src) {
14044   InstructionMark im(this);
14045   emit_int16(get_prefixq(src, dst), 0x33);
14046   emit_operand(dst, src, 0);
14047 }
14048 
14049 void Assembler::xorq(Register dst, int32_t imm32) {
14050   (void) prefixq_and_encode(dst->encoding());
14051   emit_arith(0x81, 0xF0, dst, imm32);
14052 }
14053 
14054 void Assembler::xorq(Address dst, int32_t imm32) {
14055   InstructionMark im(this);
14056   prefixq(dst);
14057   emit_arith_operand(0x81, as_Register(6), dst, imm32);
14058 }
14059 
14060 void Assembler::xorq(Address dst, Register src) {
14061   InstructionMark im(this);
14062   emit_int16(get_prefixq(dst, src), 0x31);
14063   emit_operand(src, dst, 0);
14064 }
14065 
14066 #endif // !LP64
14067 
14068 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
14069   if (VM_Version::supports_evex()) {
14070     _tuple_type = tuple_type;
14071     _input_size_in_bits = input_size_in_bits;
14072   }
14073 }