1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "asm/assembler.hpp" 26 #include "asm/assembler.inline.hpp" 27 #include "asm/codeBuffer.hpp" 28 #include "code/codeCache.hpp" 29 #include "gc/shared/cardTableBarrierSet.hpp" 30 #include "interpreter/interpreter.hpp" 31 #include "memory/resourceArea.hpp" 32 #include "memory/universe.hpp" 33 #include "prims/methodHandles.hpp" 34 #include "runtime/objectMonitor.hpp" 35 #include "runtime/os.hpp" 36 #include "runtime/sharedRuntime.hpp" 37 #include "runtime/stubRoutines.hpp" 38 #include "runtime/vm_version.hpp" 39 #include "utilities/checkedCast.hpp" 40 #include "utilities/macros.hpp" 41 42 #ifdef PRODUCT 43 #define BLOCK_COMMENT(str) /* nothing */ 44 #define STOP(error) stop(error) 45 #else 46 #define BLOCK_COMMENT(str) block_comment(str) 47 #define STOP(error) block_comment(error); stop(error) 48 #endif 49 50 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 51 // Implementation of AddressLiteral 52 53 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms. 54 static const unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = { 55 // -----------------Table 4.5 -------------------- // 56 16, 32, 64, // EVEX_FV(0) 57 4, 4, 4, // EVEX_FV(1) - with Evex.b 58 16, 32, 64, // EVEX_FV(2) - with Evex.w 59 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b 60 8, 16, 32, // EVEX_HV(0) 61 4, 4, 4, // EVEX_HV(1) - with Evex.b 62 // -----------------Table 4.6 -------------------- // 63 16, 32, 64, // EVEX_FVM(0) 64 1, 1, 1, // EVEX_T1S(0) 65 2, 2, 2, // EVEX_T1S(1) 66 4, 4, 4, // EVEX_T1S(2) 67 8, 8, 8, // EVEX_T1S(3) 68 4, 4, 4, // EVEX_T1F(0) 69 8, 8, 8, // EVEX_T1F(1) 70 8, 8, 8, // EVEX_T2(0) 71 0, 16, 16, // EVEX_T2(1) 72 0, 16, 16, // EVEX_T4(0) 73 0, 0, 32, // EVEX_T4(1) 74 0, 0, 32, // EVEX_T8(0) 75 8, 16, 32, // EVEX_HVM(0) 76 4, 8, 16, // EVEX_QVM(0) 77 2, 4, 8, // EVEX_OVM(0) 78 16, 16, 16, // EVEX_M128(0) 79 8, 32, 64, // EVEX_DUP(0) 80 1, 1, 1, // EVEX_NOSCALE(0) 81 0, 0, 0 // EVEX_ETUP 82 }; 83 84 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) { 85 _is_lval = false; 86 _target = target; 87 switch (rtype) { 88 case relocInfo::oop_type: 89 case relocInfo::metadata_type: 90 // Oops are a special case. Normally they would be their own section 91 // but in cases like icBuffer they are literals in the code stream that 92 // we don't have a section for. We use none so that we get a literal address 93 // which is always patchable. 94 break; 95 case relocInfo::external_word_type: 96 _rspec = external_word_Relocation::spec(target); 97 break; 98 case relocInfo::internal_word_type: 99 _rspec = internal_word_Relocation::spec(target); 100 break; 101 case relocInfo::opt_virtual_call_type: 102 _rspec = opt_virtual_call_Relocation::spec(); 103 break; 104 case relocInfo::static_call_type: 105 _rspec = static_call_Relocation::spec(); 106 break; 107 case relocInfo::runtime_call_type: 108 _rspec = runtime_call_Relocation::spec(); 109 break; 110 case relocInfo::poll_type: 111 case relocInfo::poll_return_type: 112 _rspec = Relocation::spec_simple(rtype); 113 break; 114 case relocInfo::none: 115 break; 116 default: 117 ShouldNotReachHere(); 118 break; 119 } 120 } 121 122 // Implementation of Address 123 124 Address Address::make_array(ArrayAddress adr) { 125 // Not implementable on 64bit machines 126 // Should have been handled higher up the call chain. 127 ShouldNotReachHere(); 128 return Address(); 129 } 130 131 // exceedingly dangerous constructor 132 Address::Address(int disp, address loc, relocInfo::relocType rtype) { 133 _base = noreg; 134 _index = noreg; 135 _scale = no_scale; 136 _disp = disp; 137 _xmmindex = xnoreg; 138 _isxmmindex = false; 139 switch (rtype) { 140 case relocInfo::external_word_type: 141 _rspec = external_word_Relocation::spec(loc); 142 break; 143 case relocInfo::internal_word_type: 144 _rspec = internal_word_Relocation::spec(loc); 145 break; 146 case relocInfo::runtime_call_type: 147 // HMM 148 _rspec = runtime_call_Relocation::spec(); 149 break; 150 case relocInfo::poll_type: 151 case relocInfo::poll_return_type: 152 _rspec = Relocation::spec_simple(rtype); 153 break; 154 case relocInfo::none: 155 break; 156 default: 157 ShouldNotReachHere(); 158 } 159 } 160 161 162 // Convert the raw encoding form into the form expected by the constructor for 163 // Address. An index of 4 (rsp) corresponds to having no index, so convert 164 // that to noreg for the Address constructor. 165 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) { 166 RelocationHolder rspec = RelocationHolder::none; 167 if (disp_reloc != relocInfo::none) { 168 rspec = Relocation::spec_simple(disp_reloc); 169 } 170 bool valid_index = index != rsp->encoding(); 171 if (valid_index) { 172 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp)); 173 madr._rspec = rspec; 174 return madr; 175 } else { 176 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp)); 177 madr._rspec = rspec; 178 return madr; 179 } 180 } 181 182 // Implementation of Assembler 183 184 int AbstractAssembler::code_fill_byte() { 185 return (u_char)'\xF4'; // hlt 186 } 187 188 void Assembler::init_attributes(void) { 189 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false); 190 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false); 191 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false); 192 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false); 193 _attributes = nullptr; 194 } 195 196 void Assembler::set_attributes(InstructionAttr* attributes) { 197 // Record the assembler in the attributes, so the attributes destructor can 198 // clear the assembler's attributes, cleaning up the otherwise dangling 199 // pointer. gcc13 has a false positive warning, because it doesn't tie that 200 // cleanup to the assignment of _attributes here. 201 attributes->set_current_assembler(this); 202 PRAGMA_DIAG_PUSH 203 PRAGMA_DANGLING_POINTER_IGNORED 204 _attributes = attributes; 205 PRAGMA_DIAG_POP 206 } 207 208 void Assembler::membar(Membar_mask_bits order_constraint) { 209 // We only have to handle StoreLoad 210 if (order_constraint & StoreLoad) { 211 // All usable chips support "locked" instructions which suffice 212 // as barriers, and are much faster than the alternative of 213 // using cpuid instruction. We use here a locked add [esp-C],0. 214 // This is conveniently otherwise a no-op except for blowing 215 // flags, and introducing a false dependency on target memory 216 // location. We can't do anything with flags, but we can avoid 217 // memory dependencies in the current method by locked-adding 218 // somewhere else on the stack. Doing [esp+C] will collide with 219 // something on stack in current method, hence we go for [esp-C]. 220 // It is convenient since it is almost always in data cache, for 221 // any small C. We need to step back from SP to avoid data 222 // dependencies with other things on below SP (callee-saves, for 223 // example). Without a clear way to figure out the minimal safe 224 // distance from SP, it makes sense to step back the complete 225 // cache line, as this will also avoid possible second-order effects 226 // with locked ops against the cache line. Our choice of offset 227 // is bounded by x86 operand encoding, which should stay within 228 // [-128; +127] to have the 8-byte displacement encoding. 229 // 230 // Any change to this code may need to revisit other places in 231 // the code where this idiom is used, in particular the 232 // orderAccess code. 233 234 int offset = -VM_Version::L1_line_size(); 235 if (offset < -128) { 236 offset = -128; 237 } 238 239 lock(); 240 addl(Address(rsp, offset), 0);// Assert the lock# signal here 241 } 242 } 243 244 // make this go away someday 245 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) { 246 if (rtype == relocInfo::none) 247 emit_int32(data); 248 else 249 emit_data(data, Relocation::spec_simple(rtype), format); 250 } 251 252 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) { 253 assert(imm_operand == 0, "default format must be immediate in this file"); 254 assert(inst_mark() != nullptr, "must be inside InstructionMark"); 255 if (rspec.type() != relocInfo::none) { 256 #ifdef ASSERT 257 check_relocation(rspec, format); 258 #endif 259 // Do not use AbstractAssembler::relocate, which is not intended for 260 // embedded words. Instead, relocate to the enclosing instruction. 261 262 // hack. call32 is too wide for mask so use disp32 263 if (format == call32_operand) 264 code_section()->relocate(inst_mark(), rspec, disp32_operand); 265 else 266 code_section()->relocate(inst_mark(), rspec, format); 267 } 268 emit_int32(data); 269 } 270 271 static int encode(Register r) { 272 return r->encoding() & 7; 273 } 274 275 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) { 276 assert(dst->has_byte_register(), "must have byte register"); 277 assert(isByte(op1) && isByte(op2), "wrong opcode"); 278 assert(isByte(imm8), "not a byte"); 279 assert((op1 & 0x01) == 0, "should be 8bit operation"); 280 emit_int24(op1, (op2 | encode(dst)), imm8); 281 } 282 283 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32, bool optimize_rax_dst) { 284 assert(isByte(op1) && isByte(op2), "wrong opcode"); 285 assert(op1 == 0x81, "Unexpected opcode"); 286 if (is8bit(imm32)) { 287 emit_int24(op1 | 0x02, // set sign bit 288 op2 | encode(dst), 289 imm32 & 0xFF); 290 } else if (optimize_rax_dst && dst == rax) { 291 switch (op2) { 292 case 0xD0: emit_int8(0x15); break; // adc 293 case 0xC0: emit_int8(0x05); break; // add 294 case 0xE0: emit_int8(0x25); break; // and 295 case 0xF8: emit_int8(0x3D); break; // cmp 296 case 0xC8: emit_int8(0x0D); break; // or 297 case 0xD8: emit_int8(0x1D); break; // sbb 298 case 0xE8: emit_int8(0x2D); break; // sub 299 case 0xF0: emit_int8(0x35); break; // xor 300 default: ShouldNotReachHere(); 301 } 302 emit_int32(imm32); 303 } else { 304 emit_int16(op1, (op2 | encode(dst))); 305 emit_int32(imm32); 306 } 307 } 308 309 // Force generation of a 4 byte immediate value even if it fits into 8bit 310 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) { 311 assert(isByte(op1) && isByte(op2), "wrong opcode"); 312 assert((op1 & 0x01) == 1, "should be 32bit operation"); 313 assert((op1 & 0x02) == 0, "sign-extension bit should not be set"); 314 emit_int16(op1, (op2 | encode(dst))); 315 emit_int32(imm32); 316 } 317 318 // immediate-to-memory forms 319 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) { 320 assert((op1 & 0x01) == 1, "should be 32bit operation"); 321 assert((op1 & 0x02) == 0, "sign-extension bit should not be set"); 322 if (is8bit(imm32)) { 323 emit_int8(op1 | 0x02); // set sign bit 324 emit_operand(rm, adr, 1); 325 emit_int8(imm32 & 0xFF); 326 } else { 327 emit_int8(op1); 328 emit_operand(rm, adr, 4); 329 emit_int32(imm32); 330 } 331 } 332 333 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) { 334 assert(op1 == 0x81, "unexpected opcode"); 335 emit_int8(op1); 336 emit_operand(rm, adr, 4); 337 emit_int32(imm32); 338 } 339 340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) { 341 assert(isByte(op1) && isByte(op2), "wrong opcode"); 342 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src))); 343 } 344 345 346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len, 347 int cur_tuple_type, int in_size_in_bits, int cur_encoding) { 348 int mod_idx = 0; 349 // We will test if the displacement fits the compressed format and if so 350 // apply the compression to the displacement iff the result is8bit. 351 if (VM_Version::supports_evex() && is_evex_inst) { 352 switch (cur_tuple_type) { 353 case EVEX_FV: 354 if ((cur_encoding & VEX_W) == VEX_W) { 355 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2; 356 } else { 357 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 358 } 359 break; 360 361 case EVEX_HV: 362 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 363 break; 364 365 case EVEX_FVM: 366 break; 367 368 case EVEX_T1S: 369 switch (in_size_in_bits) { 370 case EVEX_8bit: 371 break; 372 373 case EVEX_16bit: 374 mod_idx = 1; 375 break; 376 377 case EVEX_32bit: 378 mod_idx = 2; 379 break; 380 381 case EVEX_64bit: 382 mod_idx = 3; 383 break; 384 } 385 break; 386 387 case EVEX_T1F: 388 case EVEX_T2: 389 case EVEX_T4: 390 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0; 391 break; 392 393 case EVEX_T8: 394 break; 395 396 case EVEX_HVM: 397 break; 398 399 case EVEX_QVM: 400 break; 401 402 case EVEX_OVM: 403 break; 404 405 case EVEX_M128: 406 break; 407 408 case EVEX_DUP: 409 break; 410 411 case EVEX_NOSCALE: 412 break; 413 414 default: 415 assert(0, "no valid evex tuple_table entry"); 416 break; 417 } 418 419 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) { 420 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len]; 421 if ((disp % disp_factor) == 0) { 422 int new_disp = disp / disp_factor; 423 if ((-0x80 <= new_disp && new_disp < 0x80)) { 424 disp = new_disp; 425 } 426 } else { 427 return false; 428 } 429 } 430 } 431 return (-0x80 <= disp && disp < 0x80); 432 } 433 434 435 bool Assembler::emit_compressed_disp_byte(int &disp) { 436 int mod_idx = 0; 437 // We will test if the displacement fits the compressed format and if so 438 // apply the compression to the displacement iff the result is8bit. 439 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) { 440 int evex_encoding = _attributes->get_evex_encoding(); 441 int tuple_type = _attributes->get_tuple_type(); 442 switch (tuple_type) { 443 case EVEX_FV: 444 if ((evex_encoding & VEX_W) == VEX_W) { 445 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2; 446 } else { 447 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 448 } 449 break; 450 451 case EVEX_HV: 452 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 453 break; 454 455 case EVEX_FVM: 456 break; 457 458 case EVEX_T1S: 459 switch (_attributes->get_input_size()) { 460 case EVEX_8bit: 461 break; 462 463 case EVEX_16bit: 464 mod_idx = 1; 465 break; 466 467 case EVEX_32bit: 468 mod_idx = 2; 469 break; 470 471 case EVEX_64bit: 472 mod_idx = 3; 473 break; 474 } 475 break; 476 477 case EVEX_T1F: 478 case EVEX_T2: 479 case EVEX_T4: 480 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0; 481 break; 482 483 case EVEX_T8: 484 break; 485 486 case EVEX_HVM: 487 break; 488 489 case EVEX_QVM: 490 break; 491 492 case EVEX_OVM: 493 break; 494 495 case EVEX_M128: 496 break; 497 498 case EVEX_DUP: 499 break; 500 501 case EVEX_NOSCALE: 502 break; 503 504 default: 505 assert(0, "no valid evex tuple_table entry"); 506 break; 507 } 508 509 int vector_len = _attributes->get_vector_len(); 510 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) { 511 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len]; 512 if ((disp % disp_factor) == 0) { 513 int new_disp = disp / disp_factor; 514 if (is8bit(new_disp)) { 515 disp = new_disp; 516 } 517 } else { 518 return false; 519 } 520 } 521 } 522 return is8bit(disp); 523 } 524 525 bool Assembler::needs_rex2(Register reg1, Register reg2, Register reg3) { 526 bool rex2 = (reg1->is_valid() && reg1->encoding() >= 16) || 527 (reg2->is_valid() && reg2->encoding() >= 16) || 528 (reg3->is_valid() && reg3->encoding() >= 16); 529 assert(!rex2 || UseAPX, "extended gpr use requires UseAPX"); 530 return rex2; 531 } 532 533 #ifndef PRODUCT 534 bool Assembler::needs_evex(XMMRegister reg1, XMMRegister reg2, XMMRegister reg3) { 535 return (reg1->is_valid() && reg1->encoding() >= 16) || 536 (reg2->is_valid() && reg2->encoding() >= 16) || 537 (reg3->is_valid() && reg3->encoding() >= 16); 538 } 539 #endif 540 541 bool Assembler::needs_eevex(Register reg1, Register reg2, Register reg3) { 542 return needs_rex2(reg1, reg2, reg3); 543 } 544 545 bool Assembler::needs_eevex(int enc1, int enc2, int enc3) { 546 bool eevex = enc1 >= 16 || enc2 >= 16 || enc3 >=16; 547 assert(!eevex || UseAPX, "extended gpr use requires UseAPX"); 548 return eevex; 549 } 550 551 static bool is_valid_encoding(int reg_enc) { 552 return reg_enc >= 0; 553 } 554 555 static int raw_encode(Register reg) { 556 assert(reg == noreg || reg->is_valid(), "sanity"); 557 int reg_enc = reg->raw_encoding(); 558 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity"); 559 return reg_enc; 560 } 561 562 static int raw_encode(XMMRegister xmmreg) { 563 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity"); 564 int xmmreg_enc = xmmreg->raw_encoding(); 565 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity"); 566 return xmmreg_enc; 567 } 568 569 static int raw_encode(KRegister kreg) { 570 assert(kreg == knoreg || kreg->is_valid(), "sanity"); 571 int kreg_enc = kreg->raw_encoding(); 572 assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity"); 573 return kreg_enc; 574 } 575 576 static int modrm_encoding(int mod, int dst_enc, int src_enc) { 577 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7); 578 } 579 580 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) { 581 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7); 582 } 583 584 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) { 585 assert((mod & 3) != 0b11, "forbidden"); 586 int modrm = modrm_encoding(mod, dst_enc, src_enc); 587 emit_int8(modrm); 588 } 589 590 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc, 591 int disp) { 592 int modrm = modrm_encoding(mod, dst_enc, src_enc); 593 emit_int16(modrm, disp & 0xFF); 594 } 595 596 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc, 597 Address::ScaleFactor scale, int index_enc, int base_enc) { 598 int modrm = modrm_encoding(mod, dst_enc, src_enc); 599 int sib = sib_encoding(scale, index_enc, base_enc); 600 emit_int16(modrm, sib); 601 } 602 603 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc, 604 Address::ScaleFactor scale, int index_enc, int base_enc, 605 int disp) { 606 int modrm = modrm_encoding(mod, dst_enc, src_enc); 607 int sib = sib_encoding(scale, index_enc, base_enc); 608 emit_int24(modrm, sib, disp & 0xFF); 609 } 610 611 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc, 612 Address::ScaleFactor scale, int disp, 613 RelocationHolder const& rspec, 614 int post_addr_length) { 615 bool no_relocation = (rspec.type() == relocInfo::none); 616 617 if (is_valid_encoding(base_enc)) { 618 if (is_valid_encoding(index_enc)) { 619 assert(scale != Address::no_scale, "inconsistent address"); 620 // [base + index*scale + disp] 621 if (disp == 0 && no_relocation && ((base_enc & 0x7) != 5)) { 622 // [base + index*scale] 623 // !(rbp | r13 | r21 | r29) 624 // [00 reg 100][ss index base] 625 emit_modrm_sib(0b00, reg_enc, 0b100, 626 scale, index_enc, base_enc); 627 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 628 // [base + index*scale + imm8] 629 // [01 reg 100][ss index base] imm8 630 emit_modrm_sib_disp8(0b01, reg_enc, 0b100, 631 scale, index_enc, base_enc, 632 disp); 633 } else { 634 // [base + index*scale + disp32] 635 // [10 reg 100][ss index base] disp32 636 emit_modrm_sib(0b10, reg_enc, 0b100, 637 scale, index_enc, base_enc); 638 emit_data(disp, rspec, disp32_operand); 639 } 640 } else if ((base_enc & 0x7) == 4) { 641 // rsp | r12 | r20 | r28 642 // [rsp + disp] 643 if (disp == 0 && no_relocation) { 644 // [rsp] 645 // [00 reg 100][00 100 100] 646 emit_modrm_sib(0b00, reg_enc, 0b100, 647 Address::times_1, 0b100, 0b100); 648 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 649 // [rsp + imm8] 650 // [01 reg 100][00 100 100] disp8 651 emit_modrm_sib_disp8(0b01, reg_enc, 0b100, 652 Address::times_1, 0b100, 0b100, 653 disp); 654 } else { 655 // [rsp + imm32] 656 // [10 reg 100][00 100 100] disp32 657 emit_modrm_sib(0b10, reg_enc, 0b100, 658 Address::times_1, 0b100, 0b100); 659 emit_data(disp, rspec, disp32_operand); 660 } 661 } else { 662 // [base + disp] 663 // !(rsp | r12 | r20 | r28) were handled above 664 assert(((base_enc & 0x7) != 4), "illegal addressing mode"); 665 if (disp == 0 && no_relocation && ((base_enc & 0x7) != 5)) { 666 // [base] 667 // !(rbp | r13 | r21 | r29) 668 // [00 reg base] 669 emit_modrm(0, reg_enc, base_enc); 670 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 671 // [base + disp8] 672 // [01 reg base] disp8 673 emit_modrm_disp8(0b01, reg_enc, base_enc, 674 disp); 675 } else { 676 // [base + disp32] 677 // [10 reg base] disp32 678 emit_modrm(0b10, reg_enc, base_enc); 679 emit_data(disp, rspec, disp32_operand); 680 } 681 } 682 } else { 683 if (is_valid_encoding(index_enc)) { 684 assert(scale != Address::no_scale, "inconsistent address"); 685 // base == noreg 686 // [index*scale + disp] 687 // [00 reg 100][ss index 101] disp32 688 emit_modrm_sib(0b00, reg_enc, 0b100, 689 scale, index_enc, 0b101 /* no base */); 690 emit_data(disp, rspec, disp32_operand); 691 } else if (!no_relocation) { 692 // base == noreg, index == noreg 693 // [disp] (64bit) RIP-RELATIVE (32bit) abs 694 // [00 reg 101] disp32 695 696 emit_modrm(0b00, reg_enc, 0b101 /* no base */); 697 // Note that the RIP-rel. correction applies to the generated 698 // disp field, but _not_ to the target address in the rspec. 699 700 // disp was created by converting the target address minus the pc 701 // at the start of the instruction. That needs more correction here. 702 // intptr_t disp = target - next_ip; 703 assert(inst_mark() != nullptr, "must be inside InstructionMark"); 704 address next_ip = pc() + sizeof(int32_t) + post_addr_length; 705 int64_t adjusted = disp; 706 // Do rip-rel adjustment 707 adjusted -= (next_ip - inst_mark()); 708 assert(is_simm32(adjusted), 709 "must be 32bit offset (RIP relative address)"); 710 emit_data((int32_t) adjusted, rspec, disp32_operand); 711 712 } else { 713 // base == noreg, index == noreg, no_relocation == true 714 // 32bit never did this, did everything as the rip-rel/disp code above 715 // [disp] ABSOLUTE 716 // [00 reg 100][00 100 101] disp32 717 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */, 718 Address::times_1, 0b100, 0b101); 719 emit_data(disp, rspec, disp32_operand); 720 } 721 } 722 } 723 724 void Assembler::emit_operand(Register reg, Register base, Register index, 725 Address::ScaleFactor scale, int disp, 726 RelocationHolder const& rspec, 727 int post_addr_length) { 728 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 729 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index), 730 scale, disp, rspec, post_addr_length); 731 732 } 733 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index, 734 Address::ScaleFactor scale, int disp, 735 RelocationHolder const& rspec, 736 int post_addr_length) { 737 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 738 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported"); 739 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index), 740 scale, disp, rspec, post_addr_length); 741 } 742 743 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex, 744 Address::ScaleFactor scale, int disp, 745 RelocationHolder const& rspec, 746 int post_addr_length) { 747 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported"); 748 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported"); 749 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex), 750 scale, disp, rspec, post_addr_length); 751 } 752 753 void Assembler::emit_operand(KRegister kreg, Address adr, 754 int post_addr_length) { 755 emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp, 756 adr._rspec, 757 post_addr_length); 758 } 759 760 void Assembler::emit_operand(KRegister kreg, Register base, Register index, 761 Address::ScaleFactor scale, int disp, 762 RelocationHolder const& rspec, 763 int post_addr_length) { 764 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 765 emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index), 766 scale, disp, rspec, post_addr_length); 767 } 768 769 // Secret local extension to Assembler::WhichOperand: 770 #define end_pc_operand (_WhichOperand_limit) 771 772 address Assembler::locate_operand(address inst, WhichOperand which) { 773 // Decode the given instruction, and return the address of 774 // an embedded 32-bit operand word. 775 776 // If "which" is disp32_operand, selects the displacement portion 777 // of an effective address specifier. 778 // If "which" is imm64_operand, selects the trailing immediate constant. 779 // If "which" is call32_operand, selects the displacement of a call or jump. 780 // Caller is responsible for ensuring that there is such an operand, 781 // and that it is 32/64 bits wide. 782 783 // If "which" is end_pc_operand, find the end of the instruction. 784 785 address ip = inst; 786 bool is_64bit = false; 787 788 DEBUG_ONLY(bool has_disp32 = false); 789 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn 790 791 again_after_prefix: 792 switch (0xFF & *ip++) { 793 794 // These convenience macros generate groups of "case" labels for the switch. 795 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3 796 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \ 797 case (x)+4: case (x)+5: case (x)+6: case (x)+7 798 #define REP16(x) REP8((x)+0): \ 799 case REP8((x)+8) 800 801 case CS_segment: 802 case SS_segment: 803 case DS_segment: 804 case ES_segment: 805 case FS_segment: 806 case GS_segment: 807 // Seems dubious 808 assert(false, "shouldn't have that prefix"); 809 assert(ip == inst+1, "only one prefix allowed"); 810 goto again_after_prefix; 811 812 case 0x67: 813 case REX: 814 case REX_B: 815 case REX_X: 816 case REX_XB: 817 case REX_R: 818 case REX_RB: 819 case REX_RX: 820 case REX_RXB: 821 goto again_after_prefix; 822 823 case REX2: 824 if ((0xFF & *ip++) & REX2BIT_W) { 825 is_64bit = true; 826 } 827 goto again_after_prefix; 828 829 case REX_W: 830 case REX_WB: 831 case REX_WX: 832 case REX_WXB: 833 case REX_WR: 834 case REX_WRB: 835 case REX_WRX: 836 case REX_WRXB: 837 is_64bit = true; 838 goto again_after_prefix; 839 840 case 0xFF: // pushq a; decl a; incl a; call a; jmp a 841 case 0x88: // movb a, r 842 case 0x89: // movl a, r 843 case 0x8A: // movb r, a 844 case 0x8B: // movl r, a 845 case 0x8F: // popl a 846 DEBUG_ONLY(has_disp32 = true); 847 break; 848 849 case 0x68: // pushq #32 850 if (which == end_pc_operand) { 851 return ip + 4; 852 } 853 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate"); 854 return ip; // not produced by emit_operand 855 856 case 0x66: // movw ... (size prefix) 857 again_after_size_prefix2: 858 switch (0xFF & *ip++) { 859 case REX: 860 case REX_B: 861 case REX_X: 862 case REX_XB: 863 case REX_R: 864 case REX_RB: 865 case REX_RX: 866 case REX_RXB: 867 case REX_W: 868 case REX_WB: 869 case REX_WX: 870 case REX_WXB: 871 case REX_WR: 872 case REX_WRB: 873 case REX_WRX: 874 case REX_WRXB: 875 goto again_after_size_prefix2; 876 877 case REX2: 878 if ((0xFF & *ip++) & REX2BIT_W) { 879 is_64bit = true; 880 } 881 goto again_after_size_prefix2; 882 883 case 0x8B: // movw r, a 884 case 0x89: // movw a, r 885 DEBUG_ONLY(has_disp32 = true); 886 break; 887 case 0xC7: // movw a, #16 888 DEBUG_ONLY(has_disp32 = true); 889 tail_size = 2; // the imm16 890 break; 891 case 0x0F: // several SSE/SSE2 variants 892 ip--; // reparse the 0x0F 893 goto again_after_prefix; 894 default: 895 ShouldNotReachHere(); 896 } 897 break; 898 899 case REP8(0xB8): // movl/q r, #32/#64(oop?) 900 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4); 901 // these asserts are somewhat nonsensical 902 assert(((which == call32_operand || which == imm_operand) && is_64bit) || 903 (which == narrow_oop_operand && !is_64bit), 904 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)); 905 return ip; 906 907 case 0x69: // imul r, a, #32 908 case 0xC7: // movl a, #32(oop?) 909 tail_size = 4; 910 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 911 break; 912 913 case 0x0F: // movx..., etc. 914 switch (0xFF & *ip++) { 915 case 0x3A: // pcmpestri 916 tail_size = 1; 917 case 0x38: // ptest, pmovzxbw 918 ip++; // skip opcode 919 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 920 break; 921 922 case 0x70: // pshufd r, r/a, #8 923 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 924 case 0x73: // psrldq r, #8 925 tail_size = 1; 926 break; 927 928 case 0x10: // movups 929 case 0x11: // movups 930 case 0x12: // movlps 931 case 0x28: // movaps 932 case 0x29: // movaps 933 case 0x2E: // ucomiss 934 case 0x2F: // comiss 935 case 0x54: // andps 936 case 0x55: // andnps 937 case 0x56: // orps 938 case 0x57: // xorps 939 case 0x58: // addpd 940 case 0x59: // mulpd 941 case 0x6E: // movd 942 case 0x7E: // movd 943 case 0x6F: // movdq 944 case 0x7F: // movdq 945 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush 946 case 0xD6: // movq 947 case 0xFE: // paddd 948 DEBUG_ONLY(has_disp32 = true); 949 break; 950 951 case 0xAD: // shrd r, a, %cl 952 case 0xAF: // imul r, a 953 case 0xBE: // movsbl r, a (movsxb) 954 case 0xBF: // movswl r, a (movsxw) 955 case 0xB6: // movzbl r, a (movzxb) 956 case 0xB7: // movzwl r, a (movzxw) 957 case REP16(0x40): // cmovl cc, r, a 958 case 0xB0: // cmpxchgb 959 case 0xB1: // cmpxchg 960 case 0xC1: // xaddl 961 case 0xC7: // cmpxchg8 962 case REP16(0x90): // setcc a 963 DEBUG_ONLY(has_disp32 = true); 964 // fall out of the switch to decode the address 965 break; 966 967 case 0xC4: // pinsrw r, a, #8 968 DEBUG_ONLY(has_disp32 = true); 969 case 0xC5: // pextrw r, r, #8 970 tail_size = 1; // the imm8 971 break; 972 973 case 0xAC: // shrd r, a, #8 974 DEBUG_ONLY(has_disp32 = true); 975 tail_size = 1; // the imm8 976 break; 977 978 case REP16(0x80): // jcc rdisp32 979 if (which == end_pc_operand) return ip + 4; 980 assert(which == call32_operand, "jcc has no disp32 or imm"); 981 return ip; 982 default: 983 fatal("not handled: 0x0F%2X", 0xFF & *(ip-1)); 984 } 985 break; 986 987 case 0x81: // addl a, #32; addl r, #32 988 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl 989 // on 32bit in the case of cmpl, the imm might be an oop 990 tail_size = 4; 991 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 992 break; 993 994 case 0x83: // addl a, #8; addl r, #8 995 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl 996 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 997 tail_size = 1; 998 break; 999 1000 case 0x15: // adc rax, #32 1001 case 0x05: // add rax, #32 1002 case 0x25: // and rax, #32 1003 case 0x3D: // cmp rax, #32 1004 case 0x0D: // or rax, #32 1005 case 0x1D: // sbb rax, #32 1006 case 0x2D: // sub rax, #32 1007 case 0x35: // xor rax, #32 1008 return which == end_pc_operand ? ip + 4 : ip; 1009 1010 case 0x9B: 1011 switch (0xFF & *ip++) { 1012 case 0xD9: // fnstcw a 1013 DEBUG_ONLY(has_disp32 = true); 1014 break; 1015 default: 1016 ShouldNotReachHere(); 1017 } 1018 break; 1019 1020 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a 1021 case REP4(0x10): // adc... 1022 case REP4(0x20): // and... 1023 case REP4(0x30): // xor... 1024 case REP4(0x08): // or... 1025 case REP4(0x18): // sbb... 1026 case REP4(0x28): // sub... 1027 case 0xF7: // mull a 1028 case 0x8D: // lea r, a 1029 case 0x87: // xchg r, a 1030 case REP4(0x38): // cmp... 1031 case 0x85: // test r, a 1032 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 1033 break; 1034 1035 case 0xA8: // testb rax, #8 1036 return which == end_pc_operand ? ip + 1 : ip; 1037 case 0xA9: // testl/testq rax, #32 1038 return which == end_pc_operand ? ip + 4 : ip; 1039 1040 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8 1041 case 0xC6: // movb a, #8 1042 case 0x80: // cmpb a, #8 1043 case 0x6B: // imul r, a, #8 1044 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 1045 tail_size = 1; // the imm8 1046 break; 1047 1048 case 0xC4: // VEX_3bytes 1049 case 0xC5: // VEX_2bytes 1050 assert((UseAVX > 0), "shouldn't have VEX prefix"); 1051 assert(ip == inst+1, "no prefixes allowed"); 1052 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions 1053 // but they have prefix 0x0F and processed when 0x0F processed above. 1054 // 1055 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES 1056 // instructions (these instructions are not supported in 64-bit mode). 1057 // To distinguish them bits [7:6] are set in the VEX second byte since 1058 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set 1059 // those VEX bits REX and vvvv bits are inverted. 1060 // 1061 // Fortunately C2 doesn't generate these instructions so we don't need 1062 // to check for them in product version. 1063 1064 // Check second byte 1065 int vex_opcode; 1066 // First byte 1067 if ((0xFF & *inst) == VEX_3bytes) { 1068 vex_opcode = VEX_OPCODE_MASK & *ip; 1069 ip++; // third byte 1070 is_64bit = ((VEX_W & *ip) == VEX_W); 1071 } else { 1072 vex_opcode = VEX_OPCODE_0F; 1073 } 1074 ip++; // opcode 1075 // To find the end of instruction (which == end_pc_operand). 1076 switch (vex_opcode) { 1077 case VEX_OPCODE_0F: 1078 switch (0xFF & *ip) { 1079 case 0x70: // pshufd r, r/a, #8 1080 case 0x71: // ps[rl|ra|ll]w r, #8 1081 case 0x72: // ps[rl|ra|ll]d r, #8 1082 case 0x73: // ps[rl|ra|ll]q r, #8 1083 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8 1084 case 0xC4: // pinsrw r, r, r/a, #8 1085 case 0xC5: // pextrw r/a, r, #8 1086 case 0xC6: // shufp[s|d] r, r, r/a, #8 1087 tail_size = 1; // the imm8 1088 break; 1089 } 1090 break; 1091 case VEX_OPCODE_0F_3A: 1092 tail_size = 1; 1093 break; 1094 } 1095 ip++; // skip opcode 1096 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 1097 break; 1098 1099 case 0x62: // EVEX_4bytes 1100 assert(VM_Version::cpu_supports_evex(), "shouldn't have EVEX prefix"); 1101 assert(ip == inst+1, "no prefixes allowed"); 1102 // no EVEX collisions, all instructions that have 0x62 opcodes 1103 // have EVEX versions and are subopcodes of 0x66 1104 ip++; // skip P0 and examine W in P1 1105 is_64bit = ((VEX_W & *ip) == VEX_W); 1106 ip++; // move to P2 1107 ip++; // skip P2, move to opcode 1108 // To find the end of instruction (which == end_pc_operand). 1109 switch (0xFF & *ip) { 1110 case 0x22: // pinsrd r, r/a, #8 1111 case 0x61: // pcmpestri r, r/a, #8 1112 case 0x70: // pshufd r, r/a, #8 1113 case 0x73: // psrldq r, #8 1114 case 0x1f: // evpcmpd/evpcmpq 1115 case 0x3f: // evpcmpb/evpcmpw 1116 tail_size = 1; // the imm8 1117 break; 1118 default: 1119 break; 1120 } 1121 ip++; // skip opcode 1122 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 1123 break; 1124 1125 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1 1126 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl 1127 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a 1128 case 0xDD: // fld_d a; fst_d a; fstp_d a 1129 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a 1130 case 0xDF: // fild_d a; fistp_d a 1131 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a 1132 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a 1133 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a 1134 DEBUG_ONLY(has_disp32 = true); 1135 break; 1136 1137 case 0xE8: // call rdisp32 1138 case 0xE9: // jmp rdisp32 1139 if (which == end_pc_operand) return ip + 4; 1140 assert(which == call32_operand, "call has no disp32 or imm"); 1141 return ip; 1142 1143 case 0xF0: // Lock 1144 goto again_after_prefix; 1145 1146 case 0xF3: // For SSE 1147 case 0xF2: // For SSE2 1148 switch (0xFF & *ip++) { 1149 case REX: 1150 case REX_B: 1151 case REX_X: 1152 case REX_XB: 1153 case REX_R: 1154 case REX_RB: 1155 case REX_RX: 1156 case REX_RXB: 1157 case REX_W: 1158 case REX_WB: 1159 case REX_WX: 1160 case REX_WXB: 1161 case REX_WR: 1162 case REX_WRB: 1163 case REX_WRX: 1164 case REX_WRXB: 1165 case REX2: 1166 ip++; 1167 // fall-through 1168 default: 1169 ip++; 1170 } 1171 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands! 1172 break; 1173 1174 default: 1175 ShouldNotReachHere(); 1176 1177 #undef REP8 1178 #undef REP16 1179 } 1180 1181 assert(which != call32_operand, "instruction is not a call, jmp, or jcc"); 1182 assert(which != imm_operand, "instruction is not a movq reg, imm64"); 1183 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field"); 1184 1185 // parse the output of emit_operand 1186 int op2 = 0xFF & *ip++; 1187 int base = op2 & 0x07; 1188 int op3 = -1; 1189 const int b100 = 4; 1190 const int b101 = 5; 1191 if (base == b100 && (op2 >> 6) != 3) { 1192 op3 = 0xFF & *ip++; 1193 base = op3 & 0x07; // refetch the base 1194 } 1195 // now ip points at the disp (if any) 1196 1197 switch (op2 >> 6) { 1198 case 0: 1199 // [00 reg 100][ss index base] 1200 // [00 reg 100][00 100 esp] 1201 // [00 reg base] 1202 // [00 reg 100][ss index 101][disp32] 1203 // [00 reg 101] [disp32] 1204 1205 if (base == b101) { 1206 if (which == disp32_operand) 1207 return ip; // caller wants the disp32 1208 ip += 4; // skip the disp32 1209 } 1210 break; 1211 1212 case 1: 1213 // [01 reg 100][ss index base][disp8] 1214 // [01 reg 100][00 100 esp][disp8] 1215 // [01 reg base] [disp8] 1216 ip += 1; // skip the disp8 1217 break; 1218 1219 case 2: 1220 // [10 reg 100][ss index base][disp32] 1221 // [10 reg 100][00 100 esp][disp32] 1222 // [10 reg base] [disp32] 1223 if (which == disp32_operand) 1224 return ip; // caller wants the disp32 1225 ip += 4; // skip the disp32 1226 break; 1227 1228 case 3: 1229 // [11 reg base] (not a memory addressing mode) 1230 break; 1231 } 1232 1233 if (which == end_pc_operand) { 1234 return ip + tail_size; 1235 } 1236 1237 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32"); 1238 return ip; 1239 } 1240 1241 address Assembler::locate_next_instruction(address inst) { 1242 // Secretly share code with locate_operand: 1243 return locate_operand(inst, end_pc_operand); 1244 } 1245 1246 1247 #ifdef ASSERT 1248 void Assembler::check_relocation(RelocationHolder const& rspec, int format) { 1249 address inst = inst_mark(); 1250 assert(inst != nullptr && inst < pc(), "must point to beginning of instruction"); 1251 address opnd; 1252 1253 Relocation* r = rspec.reloc(); 1254 if (r->type() == relocInfo::none) { 1255 return; 1256 } else if (r->is_call() || format == call32_operand) { 1257 // assert(format == imm32_operand, "cannot specify a nonzero format"); 1258 opnd = locate_operand(inst, call32_operand); 1259 } else if (r->is_data()) { 1260 assert(format == imm_operand || format == disp32_operand || format == narrow_oop_operand, "format ok"); 1261 opnd = locate_operand(inst, (WhichOperand)format); 1262 } else { 1263 assert(format == imm_operand, "cannot specify a format"); 1264 return; 1265 } 1266 assert(opnd == pc(), "must put operand where relocs can find it"); 1267 } 1268 #endif // ASSERT 1269 1270 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) { 1271 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 1272 } 1273 1274 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) { 1275 if (adr.isxmmindex()) { 1276 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length); 1277 } else { 1278 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 1279 } 1280 } 1281 1282 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int byte2, int ocp_and_encoding, int byte3) { 1283 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8; 1284 if (opcode_prefix != 0) { 1285 emit_int32(opcode_prefix, (unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF), byte3); 1286 } else { 1287 emit_int24((unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF), byte3); 1288 } 1289 } 1290 1291 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int byte2, int ocp_and_encoding) { 1292 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8; 1293 if (opcode_prefix != 0) { 1294 emit_int24(opcode_prefix, (unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF)); 1295 } else { 1296 emit_int16((unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF)); 1297 } 1298 } 1299 1300 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int ocp_and_encoding) { 1301 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8; 1302 if (opcode_prefix != 0) { 1303 emit_int16(opcode_prefix, (unsigned char)byte1 | (ocp_and_encoding & 0xFF)); 1304 } else { 1305 emit_int8((unsigned char)byte1 | (ocp_and_encoding & 0xFF)); 1306 } 1307 } 1308 1309 // Now the Assembler instructions (identical for 32/64 bits) 1310 1311 void Assembler::adcl(Address dst, int32_t imm32) { 1312 InstructionMark im(this); 1313 prefix(dst); 1314 emit_arith_operand(0x81, rdx, dst, imm32); 1315 } 1316 1317 void Assembler::adcl(Address dst, Register src) { 1318 InstructionMark im(this); 1319 prefix(dst, src); 1320 emit_int8(0x11); 1321 emit_operand(src, dst, 0); 1322 } 1323 1324 void Assembler::adcl(Register dst, int32_t imm32) { 1325 prefix(dst); 1326 emit_arith(0x81, 0xD0, dst, imm32); 1327 } 1328 1329 void Assembler::adcl(Register dst, Address src) { 1330 InstructionMark im(this); 1331 prefix(src, dst); 1332 emit_int8(0x13); 1333 emit_operand(dst, src, 0); 1334 } 1335 1336 void Assembler::adcl(Register dst, Register src) { 1337 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1338 emit_arith(0x13, 0xC0, dst, src); 1339 } 1340 1341 void Assembler::addl(Address dst, int32_t imm32) { 1342 InstructionMark im(this); 1343 prefix(dst); 1344 emit_arith_operand(0x81, rax, dst, imm32); 1345 } 1346 1347 void Assembler::eaddl(Register dst, Address src, int32_t imm32, bool no_flags) { 1348 InstructionMark im(this); 1349 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1350 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1351 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 1352 emit_arith_operand(0x81, rax, src, imm32); 1353 } 1354 1355 void Assembler::addb(Address dst, int imm8) { 1356 InstructionMark im(this); 1357 prefix(dst); 1358 emit_int8((unsigned char)0x80); 1359 emit_operand(rax, dst, 1); 1360 emit_int8(imm8); 1361 } 1362 1363 void Assembler::addb(Address dst, Register src) { 1364 InstructionMark im(this); 1365 prefix(dst, src); 1366 emit_int8(0x00); 1367 emit_operand(src, dst, 0); 1368 } 1369 1370 void Assembler::addb(Register dst, int imm8) { 1371 (void) prefix_and_encode(dst->encoding(), true); 1372 emit_arith_b(0x80, 0xC0, dst, imm8); 1373 } 1374 1375 void Assembler::addw(Address dst, int imm16) { 1376 InstructionMark im(this); 1377 emit_int8(0x66); 1378 prefix(dst); 1379 emit_int8((unsigned char)0x81); 1380 emit_operand(rax, dst, 2); 1381 emit_int16(imm16); 1382 } 1383 1384 void Assembler::addw(Address dst, Register src) { 1385 InstructionMark im(this); 1386 emit_int8(0x66); 1387 prefix(dst, src); 1388 emit_int8(0x01); 1389 emit_operand(src, dst, 0); 1390 } 1391 1392 void Assembler::addl(Address dst, Register src) { 1393 InstructionMark im(this); 1394 prefix(dst, src); 1395 emit_int8(0x01); 1396 emit_operand(src, dst, 0); 1397 } 1398 1399 void Assembler::eaddl(Register dst, Address src1, Register src2, bool no_flags) { 1400 InstructionMark im(this); 1401 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */); 1402 } 1403 1404 void Assembler::addl(Register dst, int32_t imm32) { 1405 prefix(dst); 1406 emit_arith(0x81, 0xC0, dst, imm32); 1407 } 1408 1409 void Assembler::eaddl(Register dst, Register src, int32_t imm32, bool no_flags) { 1410 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xC0, no_flags); 1411 } 1412 1413 void Assembler::addl(Register dst, Address src) { 1414 InstructionMark im(this); 1415 prefix(src, dst); 1416 emit_int8(0x03); 1417 emit_operand(dst, src, 0); 1418 } 1419 1420 void Assembler::eaddl(Register dst, Register src1, Address src2, bool no_flags) { 1421 InstructionMark im(this); 1422 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, no_flags); 1423 } 1424 1425 void Assembler::addl(Register dst, Register src) { 1426 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1427 emit_arith(0x03, 0xC0, dst, src); 1428 } 1429 1430 void Assembler::eaddl(Register dst, Register src1, Register src2, bool no_flags) { 1431 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, 0xC0, no_flags, true /* is_commutative */); 1432 } 1433 1434 void Assembler::addr_nop_4() { 1435 assert(UseAddressNop, "no CPU support"); 1436 // 4 bytes: NOP DWORD PTR [EAX+0] 1437 emit_int32(0x0F, 1438 0x1F, 1439 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc); 1440 0); // 8-bits offset (1 byte) 1441 } 1442 1443 void Assembler::addr_nop_5() { 1444 assert(UseAddressNop, "no CPU support"); 1445 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset 1446 emit_int32(0x0F, 1447 0x1F, 1448 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4); 1449 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc); 1450 emit_int8(0); // 8-bits offset (1 byte) 1451 } 1452 1453 void Assembler::addr_nop_7() { 1454 assert(UseAddressNop, "no CPU support"); 1455 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset 1456 emit_int24(0x0F, 1457 0x1F, 1458 (unsigned char)0x80); 1459 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc); 1460 emit_int32(0); // 32-bits offset (4 bytes) 1461 } 1462 1463 void Assembler::addr_nop_8() { 1464 assert(UseAddressNop, "no CPU support"); 1465 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset 1466 emit_int32(0x0F, 1467 0x1F, 1468 (unsigned char)0x84, 1469 // emit_rm(cbuf, 0x2, EAX_enc, 0x4); 1470 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc); 1471 emit_int32(0); // 32-bits offset (4 bytes) 1472 } 1473 1474 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 1475 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1476 attributes.set_rex_vex_w_reverted(); 1477 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 1478 emit_int16(0x58, (0xC0 | encode)); 1479 } 1480 1481 void Assembler::addsd(XMMRegister dst, Address src) { 1482 InstructionMark im(this); 1483 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1484 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 1485 attributes.set_rex_vex_w_reverted(); 1486 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 1487 emit_int8(0x58); 1488 emit_operand(dst, src, 0); 1489 } 1490 1491 void Assembler::addss(XMMRegister dst, XMMRegister src) { 1492 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1493 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1494 emit_int16(0x58, (0xC0 | encode)); 1495 } 1496 1497 void Assembler::addss(XMMRegister dst, Address src) { 1498 InstructionMark im(this); 1499 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1500 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 1501 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1502 emit_int8(0x58); 1503 emit_operand(dst, src, 0); 1504 } 1505 1506 void Assembler::aesdec(XMMRegister dst, Address src) { 1507 assert(VM_Version::supports_aes(), ""); 1508 InstructionMark im(this); 1509 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1510 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1511 emit_int8((unsigned char)0xDE); 1512 emit_operand(dst, src, 0); 1513 } 1514 1515 void Assembler::aesdec(XMMRegister dst, XMMRegister src) { 1516 assert(VM_Version::supports_aes(), ""); 1517 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1518 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1519 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 1520 } 1521 1522 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1523 assert(VM_Version::supports_avx512_vaes(), ""); 1524 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1525 attributes.set_is_evex_instruction(); 1526 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1527 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 1528 } 1529 1530 1531 void Assembler::aesdeclast(XMMRegister dst, Address src) { 1532 assert(VM_Version::supports_aes(), ""); 1533 InstructionMark im(this); 1534 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1535 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1536 emit_int8((unsigned char)0xDF); 1537 emit_operand(dst, src, 0); 1538 } 1539 1540 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) { 1541 assert(VM_Version::supports_aes(), ""); 1542 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1543 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1544 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 1545 } 1546 1547 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1548 assert(VM_Version::supports_avx512_vaes(), ""); 1549 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1550 attributes.set_is_evex_instruction(); 1551 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1552 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 1553 } 1554 1555 void Assembler::aesenc(XMMRegister dst, Address src) { 1556 assert(VM_Version::supports_aes(), ""); 1557 InstructionMark im(this); 1558 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1559 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1560 emit_int8((unsigned char)0xDC); 1561 emit_operand(dst, src, 0); 1562 } 1563 1564 void Assembler::aesenc(XMMRegister dst, XMMRegister src) { 1565 assert(VM_Version::supports_aes(), ""); 1566 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1567 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1568 emit_int16((unsigned char)0xDC, 0xC0 | encode); 1569 } 1570 1571 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1572 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling"); 1573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1574 attributes.set_is_evex_instruction(); 1575 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1576 emit_int16((unsigned char)0xDC, (0xC0 | encode)); 1577 } 1578 1579 void Assembler::aesenclast(XMMRegister dst, Address src) { 1580 assert(VM_Version::supports_aes(), ""); 1581 InstructionMark im(this); 1582 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1583 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1584 emit_int8((unsigned char)0xDD); 1585 emit_operand(dst, src, 0); 1586 } 1587 1588 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) { 1589 assert(VM_Version::supports_aes(), ""); 1590 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1591 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1592 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 1593 } 1594 1595 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1596 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling"); 1597 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1598 attributes.set_is_evex_instruction(); 1599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1600 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 1601 } 1602 1603 void Assembler::andb(Address dst, Register src) { 1604 InstructionMark im(this); 1605 prefix(dst, src, true); 1606 emit_int8(0x20); 1607 emit_operand(src, dst, 0); 1608 } 1609 1610 void Assembler::andl(Address dst, int32_t imm32) { 1611 InstructionMark im(this); 1612 prefix(dst); 1613 emit_arith_operand(0x81, as_Register(4), dst, imm32); 1614 } 1615 1616 void Assembler::eandl(Register dst, Address src, int32_t imm32, bool no_flags) { 1617 InstructionMark im(this); 1618 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1619 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1620 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 1621 emit_arith_operand(0x81, rsp, src, imm32); 1622 } 1623 1624 void Assembler::andl(Register dst, int32_t imm32) { 1625 prefix(dst); 1626 emit_arith(0x81, 0xE0, dst, imm32); 1627 } 1628 1629 void Assembler::eandl(Register dst, Register src, int32_t imm32, bool no_flags) { 1630 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xE0, no_flags); 1631 } 1632 1633 void Assembler::andl(Address dst, Register src) { 1634 InstructionMark im(this); 1635 prefix(dst, src); 1636 emit_int8(0x21); 1637 emit_operand(src, dst, 0); 1638 } 1639 1640 void Assembler::andl(Register dst, Address src) { 1641 InstructionMark im(this); 1642 prefix(src, dst); 1643 emit_int8(0x23); 1644 emit_operand(dst, src, 0); 1645 } 1646 1647 void Assembler::eandl(Register dst, Register src1, Address src2, bool no_flags) { 1648 InstructionMark im(this); 1649 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, no_flags); 1650 } 1651 1652 void Assembler::eandl(Register dst, Address src1, Register src2, bool no_flags) { 1653 InstructionMark im(this); 1654 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */); 1655 } 1656 1657 void Assembler::andl(Register dst, Register src) { 1658 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1659 emit_arith(0x23, 0xC0, dst, src); 1660 } 1661 1662 void Assembler::eandl(Register dst, Register src1, Register src2, bool no_flags) { 1663 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, 0xC0, no_flags, true /* is_commutative */); 1664 } 1665 1666 void Assembler::andnl(Register dst, Register src1, Register src2) { 1667 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1668 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1669 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 1670 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 1671 } 1672 1673 void Assembler::andnl(Register dst, Register src1, Address src2) { 1674 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1675 InstructionMark im(this); 1676 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1677 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1678 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1679 emit_int8((unsigned char)0xF2); 1680 emit_operand(dst, src2, 0); 1681 } 1682 1683 void Assembler::bsfl(Register dst, Register src) { 1684 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 1685 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode); 1686 } 1687 1688 void Assembler::bsrl(Register dst, Register src) { 1689 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 1690 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode); 1691 } 1692 1693 void Assembler::bswapl(Register reg) { // bswap 1694 int encode = prefix_and_encode(reg->encoding(), false, true /* is_map1 */); 1695 emit_opcode_prefix_and_encoding((unsigned char)0xC8, encode); 1696 } 1697 1698 void Assembler::blsil(Register dst, Register src) { 1699 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1700 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1701 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 1702 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 1703 } 1704 1705 void Assembler::blsil(Register dst, Address src) { 1706 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1707 InstructionMark im(this); 1708 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1709 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1710 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1711 emit_int8((unsigned char)0xF3); 1712 emit_operand(rbx, src, 0); 1713 } 1714 1715 void Assembler::blsmskl(Register dst, Register src) { 1716 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1717 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1718 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 1719 emit_int16((unsigned char)0xF3, 1720 0xC0 | encode); 1721 } 1722 1723 void Assembler::blsmskl(Register dst, Address src) { 1724 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1725 InstructionMark im(this); 1726 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1727 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1728 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1729 emit_int8((unsigned char)0xF3); 1730 emit_operand(rdx, src, 0); 1731 } 1732 1733 void Assembler::blsrl(Register dst, Register src) { 1734 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1735 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1736 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 1737 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 1738 } 1739 1740 void Assembler::blsrl(Register dst, Address src) { 1741 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1742 InstructionMark im(this); 1743 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1744 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 1745 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1746 emit_int8((unsigned char)0xF3); 1747 emit_operand(rcx, src, 0); 1748 } 1749 1750 void Assembler::call(Label& L, relocInfo::relocType rtype) { 1751 if (L.is_bound()) { 1752 const int long_size = 5; 1753 int offs = (int)( target(L) - pc() ); 1754 assert(offs <= 0, "assembler error"); 1755 InstructionMark im(this); 1756 // 1110 1000 #32-bit disp 1757 emit_int8((unsigned char)0xE8); 1758 emit_data(offs - long_size, rtype, disp32_operand); 1759 } else { 1760 InstructionMark im(this); 1761 // 1110 1000 #32-bit disp 1762 L.add_patch_at(code(), locator()); 1763 1764 emit_int8((unsigned char)0xE8); 1765 emit_data(int(0), rtype, disp32_operand); 1766 } 1767 } 1768 1769 void Assembler::call(Register dst) { 1770 int encode = prefix_and_encode(dst->encoding()); 1771 emit_int16((unsigned char)0xFF, (0xD0 | encode)); 1772 } 1773 1774 1775 void Assembler::call(Address adr) { 1776 assert(!adr._rspec.reloc()->is_data(), "should not use ExternalAddress for call"); 1777 InstructionMark im(this); 1778 prefix(adr); 1779 emit_int8((unsigned char)0xFF); 1780 emit_operand(rdx, adr, 0); 1781 } 1782 1783 void Assembler::call_literal(address entry, RelocationHolder const& rspec) { 1784 InstructionMark im(this); 1785 emit_int8((unsigned char)0xE8); 1786 intptr_t disp = entry - (pc() + sizeof(int32_t)); 1787 // Entry is null in case of a scratch emit. 1788 assert(entry == nullptr || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp); 1789 // Technically, should use call32_operand, but this format is 1790 // implied by the fact that we're emitting a call instruction. 1791 1792 emit_data((int) disp, rspec, disp32_operand); 1793 } 1794 1795 void Assembler::cdql() { 1796 emit_int8((unsigned char)0x99); 1797 } 1798 1799 void Assembler::cld() { 1800 emit_int8((unsigned char)0xFC); 1801 } 1802 1803 void Assembler::cmovl(Condition cc, Register dst, Register src) { 1804 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 1805 emit_opcode_prefix_and_encoding(0x40 | cc, 0xC0, encode); 1806 } 1807 1808 void Assembler::ecmovl(Condition cc, Register dst, Register src1, Register src2) { 1809 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x40 | cc, false /* no_flags */, true /* is_map1 */, true /* swap */); 1810 } 1811 1812 void Assembler::cmovl(Condition cc, Register dst, Address src) { 1813 InstructionMark im(this); 1814 prefix(src, dst, false, true /* is_map1 */); 1815 emit_int8((0x40 | cc)); 1816 emit_operand(dst, src, 0); 1817 } 1818 1819 void Assembler::ecmovl(Condition cc, Register dst, Register src1, Address src2) { 1820 InstructionMark im(this); 1821 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, (0x40 | cc) , false /* no_flags */, true /* is_map1 */); 1822 } 1823 1824 void Assembler::cmpb(Address dst, Register reg) { 1825 assert(reg->has_byte_register(), "must have byte register"); 1826 InstructionMark im(this); 1827 prefix(dst, reg, true); 1828 emit_int8((unsigned char)0x38); 1829 emit_operand(reg, dst, 0); 1830 } 1831 1832 void Assembler::cmpb(Register reg, Address dst) { 1833 assert(reg->has_byte_register(), "must have byte register"); 1834 InstructionMark im(this); 1835 prefix(dst, reg, true); 1836 emit_int8((unsigned char)0x3a); 1837 emit_operand(reg, dst, 0); 1838 } 1839 1840 void Assembler::cmpb(Address dst, int imm8) { 1841 InstructionMark im(this); 1842 prefix(dst); 1843 emit_int8((unsigned char)0x80); 1844 emit_operand(rdi, dst, 1); 1845 emit_int8(imm8); 1846 } 1847 1848 void Assembler::cmpb(Register dst, int imm8) { 1849 prefix(dst); 1850 emit_arith_b(0x80, 0xF8, dst, imm8); 1851 } 1852 1853 void Assembler::cmpl(Address dst, int32_t imm32) { 1854 InstructionMark im(this); 1855 prefix(dst); 1856 emit_arith_operand(0x81, as_Register(7), dst, imm32); 1857 } 1858 1859 void Assembler::cmpl(Register dst, int32_t imm32) { 1860 prefix(dst); 1861 emit_arith(0x81, 0xF8, dst, imm32); 1862 } 1863 1864 void Assembler::cmpl(Register dst, Register src) { 1865 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1866 emit_arith(0x3B, 0xC0, dst, src); 1867 } 1868 1869 void Assembler::cmpl(Register dst, Address src) { 1870 InstructionMark im(this); 1871 prefix(src, dst); 1872 emit_int8(0x3B); 1873 emit_operand(dst, src, 0); 1874 } 1875 1876 void Assembler::cmpl(Address dst, Register reg) { 1877 InstructionMark im(this); 1878 prefix(dst, reg); 1879 emit_int8(0x39); 1880 emit_operand(reg, dst, 0); 1881 } 1882 1883 void Assembler::cmpl_imm32(Address dst, int32_t imm32) { 1884 InstructionMark im(this); 1885 prefix(dst); 1886 emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32); 1887 } 1888 1889 void Assembler::cmpw(Address dst, int imm16) { 1890 InstructionMark im(this); 1891 emit_int8(0x66); 1892 prefix(dst); 1893 emit_int8((unsigned char)0x81); 1894 emit_operand(rdi, dst, 2); 1895 emit_int16(imm16); 1896 } 1897 1898 void Assembler::cmpw(Address dst, Register reg) { 1899 InstructionMark im(this); 1900 emit_int8(0x66); 1901 prefix(dst, reg); 1902 emit_int8((unsigned char)0x39); 1903 emit_operand(reg, dst, 0); 1904 } 1905 1906 // The 32-bit cmpxchg compares the value at adr with the contents of rax, 1907 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,. 1908 // The ZF is set if the compared values were equal, and cleared otherwise. 1909 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg 1910 InstructionMark im(this); 1911 prefix(adr, reg, false, true /* is_map1 */); 1912 emit_int8((unsigned char)0xB1); 1913 emit_operand(reg, adr, 0); 1914 } 1915 1916 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg 1917 InstructionMark im(this); 1918 size_prefix(); 1919 prefix(adr, reg, false, true /* is_map1 */); 1920 emit_int8((unsigned char)0xB1); 1921 emit_operand(reg, adr, 0); 1922 } 1923 1924 // The 8-bit cmpxchg compares the value at adr with the contents of rax, 1925 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,. 1926 // The ZF is set if the compared values were equal, and cleared otherwise. 1927 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg 1928 InstructionMark im(this); 1929 prefix(adr, reg, true, true /* is_map1 */); 1930 emit_int8((unsigned char)0xB0); 1931 emit_operand(reg, adr, 0); 1932 } 1933 1934 void Assembler::comisd(XMMRegister dst, Address src) { 1935 // NOTE: dbx seems to decode this as comiss even though the 1936 // 0x66 is there. Strangely ucomisd comes out correct 1937 InstructionMark im(this); 1938 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);; 1939 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 1940 attributes.set_rex_vex_w_reverted(); 1941 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 1942 emit_int8(0x2F); 1943 emit_operand(dst, src, 0); 1944 } 1945 1946 void Assembler::comisd(XMMRegister dst, XMMRegister src) { 1947 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1948 attributes.set_rex_vex_w_reverted(); 1949 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 1950 emit_int16(0x2F, (0xC0 | encode)); 1951 } 1952 1953 void Assembler::comiss(XMMRegister dst, Address src) { 1954 InstructionMark im(this); 1955 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1956 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 1957 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1958 emit_int8(0x2F); 1959 emit_operand(dst, src, 0); 1960 } 1961 1962 void Assembler::comiss(XMMRegister dst, XMMRegister src) { 1963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1964 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1965 emit_int16(0x2F, (0xC0 | encode)); 1966 } 1967 1968 void Assembler::cpuid() { 1969 emit_int16(0x0F, (unsigned char)0xA2); 1970 } 1971 1972 void Assembler::serialize() { 1973 assert(VM_Version::supports_serialize(), ""); 1974 emit_int24(0x0F, 0x01, 0xE8); 1975 } 1976 1977 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented 1978 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v 1979 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. - 1980 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. - 1981 // 1982 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v 1983 // 1984 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v 1985 // 1986 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v 1987 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) { 1988 assert(VM_Version::supports_sse4_2(), ""); 1989 if (needs_eevex(crc, v)) { 1990 InstructionAttr attributes(AVX_128bit, /* rex_w */ sizeInBytes == 8, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1991 int encode = vex_prefix_and_encode(crc->encoding(), 0, v->encoding(), sizeInBytes == 2 ? VEX_SIMD_66 : VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true); 1992 emit_int16(sizeInBytes == 1 ? (unsigned char)0xF0 : (unsigned char)0xF1, (0xC0 | encode)); 1993 } else { 1994 int8_t w = 0x01; 1995 Prefix p = Prefix_EMPTY; 1996 1997 emit_int8((unsigned char)0xF2); 1998 switch (sizeInBytes) { 1999 case 1: 2000 w = 0; 2001 break; 2002 case 2: 2003 case 4: 2004 break; 2005 case 8: 2006 // Note: 2007 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf 2008 // 2009 // Page B - 72 Vol. 2C says 2010 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2 2011 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m 2012 // F0!!! 2013 // while 3 - 208 Vol. 2A 2014 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64. 2015 // 2016 // the 0 on a last bit is reserved for a different flavor of this instruction : 2017 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8. 2018 p = REX_W; 2019 break; 2020 default: 2021 assert(0, "Unsupported value for a sizeInBytes argument"); 2022 break; 2023 } 2024 prefix(crc, v, p); 2025 emit_int32(0x0F, 2026 0x38, 2027 0xF0 | w, 2028 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7)); 2029 } 2030 } 2031 2032 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) { 2033 assert(VM_Version::supports_sse4_2(), ""); 2034 InstructionMark im(this); 2035 if (needs_eevex(crc, adr.base(), adr.index())) { 2036 InstructionAttr attributes(AVX_128bit, /* vex_w */ sizeInBytes == 8, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2037 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 2038 vex_prefix(adr, 0, crc->encoding(), sizeInBytes == 2 ? VEX_SIMD_66 : VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes); 2039 emit_int8(sizeInBytes == 1 ? (unsigned char)0xF0 : (unsigned char)0xF1); 2040 emit_operand(crc, adr, 0); 2041 } else { 2042 int8_t w = 0x01; 2043 Prefix p = Prefix_EMPTY; 2044 2045 emit_int8((uint8_t)0xF2); 2046 switch (sizeInBytes) { 2047 case 1: 2048 w = 0; 2049 break; 2050 case 2: 2051 case 4: 2052 break; 2053 case 8: 2054 p = REX_W; 2055 break; 2056 default: 2057 assert(0, "Unsupported value for a sizeInBytes argument"); 2058 break; 2059 } 2060 prefix(crc, adr, p); 2061 emit_int24(0x0F, 0x38, (0xF0 | w)); 2062 emit_operand(crc, adr, 0); 2063 } 2064 } 2065 2066 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) { 2067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2068 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2069 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2070 } 2071 2072 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) { 2073 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2074 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2075 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2076 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2077 } 2078 2079 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 2080 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 2081 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true); 2082 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2083 emit_int24(0x1D, (0xC0 | encode), imm8); 2084 } 2085 2086 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) { 2087 assert(VM_Version::supports_evex(), ""); 2088 InstructionMark im(this); 2089 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true); 2090 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit); 2091 attributes.reset_is_clear_context(); 2092 attributes.set_embedded_opmask_register_specifier(mask); 2093 attributes.set_is_evex_instruction(); 2094 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2095 emit_int8(0x1D); 2096 emit_operand(src, dst, 1); 2097 emit_int8(imm8); 2098 } 2099 2100 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) { 2101 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 2102 InstructionMark im(this); 2103 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true); 2104 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 2105 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2106 emit_int8(0x1D); 2107 emit_operand(src, dst, 1); 2108 emit_int8(imm8); 2109 } 2110 2111 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2112 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 2113 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true); 2114 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2115 emit_int16(0x13, (0xC0 | encode)); 2116 } 2117 2118 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) { 2119 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 2120 InstructionMark im(this); 2121 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true); 2122 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 2123 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2124 emit_int8(0x13); 2125 emit_operand(dst, src, 0); 2126 } 2127 2128 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) { 2129 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2130 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2131 emit_int16(0x5B, (0xC0 | encode)); 2132 } 2133 2134 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2135 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2136 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2137 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2138 emit_int16(0x5B, (0xC0 | encode)); 2139 } 2140 2141 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 2142 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2143 attributes.set_rex_vex_w_reverted(); 2144 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2145 emit_int16(0x5A, (0xC0 | encode)); 2146 } 2147 2148 void Assembler::cvtsd2ss(XMMRegister dst, Address src) { 2149 InstructionMark im(this); 2150 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2151 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 2152 attributes.set_rex_vex_w_reverted(); 2153 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2154 emit_int8(0x5A); 2155 emit_operand(dst, src, 0); 2156 } 2157 2158 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) { 2159 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2160 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true); 2161 emit_int16(0x2A, (0xC0 | encode)); 2162 } 2163 2164 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) { 2165 InstructionMark im(this); 2166 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2167 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2168 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2169 emit_int8(0x2A); 2170 emit_operand(dst, src, 0); 2171 } 2172 2173 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) { 2174 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2175 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes, true); 2176 emit_int16(0x2A, (0xC0 | encode)); 2177 } 2178 2179 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) { 2180 InstructionMark im(this); 2181 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2182 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2183 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2184 emit_int8(0x2A); 2185 emit_operand(dst, src, 0); 2186 } 2187 2188 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) { 2189 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2190 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes, true); 2191 emit_int16(0x2A, (0xC0 | encode)); 2192 } 2193 2194 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 2195 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2196 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2197 emit_int16(0x5A, (0xC0 | encode)); 2198 } 2199 2200 void Assembler::cvtss2sd(XMMRegister dst, Address src) { 2201 InstructionMark im(this); 2202 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2203 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2204 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2205 emit_int8(0x5A); 2206 emit_operand(dst, src, 0); 2207 } 2208 2209 2210 void Assembler::cvttsd2sil(Register dst, XMMRegister src) { 2211 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2212 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2213 emit_int16(0x2C, (0xC0 | encode)); 2214 } 2215 2216 void Assembler::cvtss2sil(Register dst, XMMRegister src) { 2217 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2218 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2219 emit_int16(0x2D, (0xC0 | encode)); 2220 } 2221 2222 void Assembler::cvttss2sil(Register dst, XMMRegister src) { 2223 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2224 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2225 emit_int16(0x2C, (0xC0 | encode)); 2226 } 2227 2228 void Assembler::evcvttss2sisl(Register dst, XMMRegister src) { 2229 assert(VM_Version::supports_avx10_2(), ""); 2230 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2231 attributes.set_is_evex_instruction(); 2232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 2233 emit_int16(0x6D, (0xC0 | encode)); 2234 } 2235 2236 void Assembler::evcvttss2sisl(Register dst, Address src) { 2237 assert(VM_Version::supports_avx10_2(), ""); 2238 InstructionMark im(this); 2239 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2240 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2241 attributes.set_is_evex_instruction(); 2242 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 2243 emit_int8((unsigned char)0x6D); 2244 emit_operand(dst, src, 0); 2245 } 2246 2247 void Assembler::evcvttss2sisq(Register dst, XMMRegister src) { 2248 assert(VM_Version::supports_avx10_2(), ""); 2249 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2250 attributes.set_is_evex_instruction(); 2251 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 2252 emit_int16(0x6D, (0xC0 | encode)); 2253 } 2254 2255 void Assembler::evcvttss2sisq(Register dst, Address src) { 2256 assert(VM_Version::supports_avx10_2(), ""); 2257 InstructionMark im(this); 2258 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2259 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2260 attributes.set_is_evex_instruction(); 2261 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 2262 emit_int8((unsigned char)0x6D); 2263 emit_operand(dst, src, 0); 2264 } 2265 2266 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) { 2267 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2268 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2269 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2270 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2271 } 2272 2273 void Assembler::pabsb(XMMRegister dst, XMMRegister src) { 2274 assert(VM_Version::supports_ssse3(), ""); 2275 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2276 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2277 emit_int16(0x1C, (0xC0 | encode)); 2278 } 2279 2280 void Assembler::pabsw(XMMRegister dst, XMMRegister src) { 2281 assert(VM_Version::supports_ssse3(), ""); 2282 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2283 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2284 emit_int16(0x1D, (0xC0 | encode)); 2285 } 2286 2287 void Assembler::pabsd(XMMRegister dst, XMMRegister src) { 2288 assert(VM_Version::supports_ssse3(), ""); 2289 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2290 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2291 emit_int16(0x1E, (0xC0 | encode)); 2292 } 2293 2294 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) { 2295 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 2296 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 2297 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported"); 2298 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2299 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2300 emit_int16(0x1C, (0xC0 | encode)); 2301 } 2302 2303 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) { 2304 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 2305 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 2306 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, ""); 2307 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2308 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2309 emit_int16(0x1D, (0xC0 | encode)); 2310 } 2311 2312 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) { 2313 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 2314 vector_len == AVX_256bit? VM_Version::supports_avx2() : 2315 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, ""); 2316 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2317 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2318 emit_int16(0x1E, (0xC0 | encode)); 2319 } 2320 2321 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) { 2322 assert(UseAVX > 2, ""); 2323 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2324 attributes.set_is_evex_instruction(); 2325 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2326 emit_int16(0x1F, (0xC0 | encode)); 2327 } 2328 2329 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) { 2330 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2331 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2332 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2333 emit_int16(0x5A, (0xC0 | encode)); 2334 } 2335 2336 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2337 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2338 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2339 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2340 attributes.set_rex_vex_w_reverted(); 2341 emit_int16(0x5A, (0xC0 | encode)); 2342 } 2343 2344 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2345 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2346 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2347 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2348 emit_int16(0x5B, (0xC0 | encode)); 2349 } 2350 2351 void Assembler::evcvttps2dqs(XMMRegister dst, XMMRegister src, int vector_len) { 2352 assert(VM_Version::supports_avx10_2(), ""); 2353 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2354 attributes.set_is_evex_instruction(); 2355 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 2356 emit_int16(0x6D, (0xC0 | encode)); 2357 } 2358 2359 void Assembler::evcvttps2dqs(XMMRegister dst, Address src, int vector_len) { 2360 assert(VM_Version::supports_avx10_2(), ""); 2361 InstructionMark im(this); 2362 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2363 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 2364 attributes.set_is_evex_instruction(); 2365 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 2366 emit_int8((unsigned char)0x6D); 2367 emit_operand(dst, src, 0); 2368 } 2369 2370 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2371 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2372 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2373 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2374 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2375 } 2376 2377 void Assembler::evcvttpd2dqs(XMMRegister dst, XMMRegister src, int vector_len) { 2378 assert(VM_Version::supports_avx10_2(), ""); 2379 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2380 attributes.set_is_evex_instruction(); 2381 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 2382 emit_int16(0x6D, (0xC0 | encode)); 2383 } 2384 2385 void Assembler::evcvttpd2dqs(XMMRegister dst, Address src, int vector_len) { 2386 assert(VM_Version::supports_avx10_2(), ""); 2387 InstructionMark im(this); 2388 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2389 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 2390 attributes.set_is_evex_instruction(); 2391 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 2392 emit_int8((unsigned char)0x6D); 2393 emit_operand(dst, src, 0); 2394 } 2395 2396 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2397 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2398 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2399 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2400 emit_int16(0x5B, (0xC0 | encode)); 2401 } 2402 2403 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2404 assert(VM_Version::supports_avx512dq(), ""); 2405 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2406 attributes.set_is_evex_instruction(); 2407 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2408 emit_int16(0x7A, (0xC0 | encode)); 2409 } 2410 2411 void Assembler::evcvttps2qqs(XMMRegister dst, XMMRegister src, int vector_len) { 2412 assert(VM_Version::supports_avx10_2(), ""); 2413 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2414 attributes.set_is_evex_instruction(); 2415 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes); 2416 emit_int16(0x6D, (0xC0 | encode)); 2417 } 2418 2419 void Assembler::evcvttps2qqs(XMMRegister dst, Address src, int vector_len) { 2420 assert(VM_Version::supports_avx10_2(), ""); 2421 InstructionMark im(this); 2422 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2423 attributes.set_address_attributes(/* tuple_type */ EVEX_HV, /* input_size_in_bits */ EVEX_32bit); 2424 attributes.set_is_evex_instruction(); 2425 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes); 2426 emit_int8((unsigned char)0x6D); 2427 emit_operand(dst, src, 0); 2428 } 2429 2430 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2431 assert(VM_Version::supports_avx512dq(), ""); 2432 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2433 attributes.set_is_evex_instruction(); 2434 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2435 emit_int16(0x7B, (0xC0 | encode)); 2436 } 2437 2438 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2439 assert(VM_Version::supports_avx512dq(), ""); 2440 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2441 attributes.set_is_evex_instruction(); 2442 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2443 emit_int16(0x5B, (0xC0 | encode)); 2444 } 2445 2446 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2447 assert(VM_Version::supports_avx512dq(), ""); 2448 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2449 attributes.set_is_evex_instruction(); 2450 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2451 emit_int16(0x7A, (0xC0 | encode)); 2452 } 2453 2454 void Assembler::evcvttpd2qqs(XMMRegister dst, XMMRegister src, int vector_len) { 2455 assert(VM_Version::supports_avx10_2(), ""); 2456 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2457 attributes.set_is_evex_instruction(); 2458 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes); 2459 emit_int16(0x6D, (0xC0 | encode)); 2460 } 2461 2462 void Assembler::evcvttpd2qqs(XMMRegister dst, Address src, int vector_len) { 2463 assert(VM_Version::supports_avx10_2(), ""); 2464 InstructionMark im(this); 2465 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2466 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 2467 attributes.set_is_evex_instruction(); 2468 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes); 2469 emit_int8((unsigned char)0x6D); 2470 emit_operand(dst, src, 0); 2471 } 2472 2473 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) { 2474 assert(VM_Version::supports_avx512dq(), ""); 2475 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2476 attributes.set_is_evex_instruction(); 2477 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2478 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2479 } 2480 2481 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) { 2482 assert(VM_Version::supports_avx512bw(), ""); 2483 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2484 attributes.set_is_evex_instruction(); 2485 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2486 emit_int16(0x30, (0xC0 | encode)); 2487 } 2488 2489 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) { 2490 assert(UseAVX > 2, ""); 2491 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2492 attributes.set_is_evex_instruction(); 2493 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2494 emit_int16(0x33, (0xC0 | encode)); 2495 } 2496 2497 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) { 2498 assert(UseAVX > 2, ""); 2499 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2500 attributes.set_is_evex_instruction(); 2501 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2502 emit_int16(0x31, (0xC0 | encode)); 2503 } 2504 2505 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) { 2506 assert(UseAVX > 2, ""); 2507 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2508 attributes.set_is_evex_instruction(); 2509 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2510 emit_int16(0x35, (0xC0 | encode)); 2511 } 2512 2513 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) { 2514 assert(UseAVX > 2, ""); 2515 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2516 attributes.set_is_evex_instruction(); 2517 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2518 emit_int16(0x32, (0xC0 | encode)); 2519 } 2520 2521 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) { 2522 assert(UseAVX > 2, ""); 2523 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2524 attributes.set_is_evex_instruction(); 2525 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2526 emit_int16(0x34, (0xC0 | encode)); 2527 } 2528 2529 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) { 2530 assert(UseAVX > 2, ""); 2531 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2532 attributes.set_is_evex_instruction(); 2533 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2534 emit_int16(0x25, (0xC0 | encode)); 2535 } 2536 2537 void Assembler::decl(Address dst) { 2538 // Don't use it directly. Use MacroAssembler::decrement() instead. 2539 InstructionMark im(this); 2540 prefix(dst); 2541 emit_int8((unsigned char)0xFF); 2542 emit_operand(rcx, dst, 0); 2543 } 2544 2545 void Assembler::edecl(Register dst, Address src, bool no_flags) { 2546 InstructionMark im(this); 2547 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2548 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 2549 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2550 emit_int8((unsigned char)0xFF); 2551 emit_operand(rcx, src, 0); 2552 } 2553 2554 void Assembler::divsd(XMMRegister dst, Address src) { 2555 InstructionMark im(this); 2556 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2557 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 2558 attributes.set_rex_vex_w_reverted(); 2559 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2560 emit_int8(0x5E); 2561 emit_operand(dst, src, 0); 2562 } 2563 2564 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 2565 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2566 attributes.set_rex_vex_w_reverted(); 2567 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2568 emit_int16(0x5E, (0xC0 | encode)); 2569 } 2570 2571 void Assembler::divss(XMMRegister dst, Address src) { 2572 InstructionMark im(this); 2573 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2574 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2575 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2576 emit_int8(0x5E); 2577 emit_operand(dst, src, 0); 2578 } 2579 2580 void Assembler::divss(XMMRegister dst, XMMRegister src) { 2581 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2582 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2583 emit_int16(0x5E, (0xC0 | encode)); 2584 } 2585 2586 void Assembler::hlt() { 2587 emit_int8((unsigned char)0xF4); 2588 } 2589 2590 void Assembler::idivl(Register src) { 2591 int encode = prefix_and_encode(src->encoding()); 2592 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 2593 } 2594 2595 void Assembler::eidivl(Register src, bool no_flags) { // Signed 2596 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2597 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2598 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 2599 } 2600 2601 void Assembler::divl(Register src) { // Unsigned 2602 int encode = prefix_and_encode(src->encoding()); 2603 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 2604 } 2605 2606 void Assembler::edivl(Register src, bool no_flags) { // Unsigned 2607 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2608 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2609 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 2610 } 2611 2612 void Assembler::imull(Register src) { 2613 int encode = prefix_and_encode(src->encoding()); 2614 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 2615 } 2616 2617 void Assembler::eimull(Register src, bool no_flags) { 2618 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2619 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2620 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 2621 } 2622 2623 void Assembler::imull(Register dst, Register src) { 2624 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 2625 emit_opcode_prefix_and_encoding((unsigned char)0xAF, 0xC0, encode); 2626 } 2627 2628 void Assembler::eimull(Register dst, Register src1, Register src2, bool no_flags) { 2629 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */); 2630 } 2631 2632 void Assembler::imull(Register dst, Address src, int32_t value) { 2633 InstructionMark im(this); 2634 prefix(src, dst); 2635 if (is8bit(value)) { 2636 emit_int8((unsigned char)0x6B); 2637 emit_operand(dst, src, 1); 2638 emit_int8(value); 2639 } else { 2640 emit_int8((unsigned char)0x69); 2641 emit_operand(dst, src, 4); 2642 emit_int32(value); 2643 } 2644 } 2645 2646 void Assembler::eimull(Register dst, Address src, int32_t value, bool no_flags) { 2647 InstructionMark im(this); 2648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2649 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 2650 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2651 if (is8bit(value)) { 2652 emit_int8((unsigned char)0x6B); 2653 emit_operand(dst, src, 1); 2654 emit_int8(value); 2655 } else { 2656 emit_int8((unsigned char)0x69); 2657 emit_operand(dst, src, 4); 2658 emit_int32(value); 2659 } 2660 } 2661 2662 void Assembler::imull(Register dst, Register src, int value) { 2663 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 2664 if (is8bit(value)) { 2665 emit_int24(0x6B, (0xC0 | encode), value & 0xFF); 2666 } else { 2667 emit_int16(0x69, (0xC0 | encode)); 2668 emit_int32(value); 2669 } 2670 } 2671 2672 void Assembler::eimull(Register dst, Register src, int value, bool no_flags) { 2673 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2674 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2675 if (is8bit(value)) { 2676 emit_int24(0x6B, (0xC0 | encode), value & 0xFF); 2677 } else { 2678 emit_int16(0x69, (0xC0 | encode)); 2679 emit_int32(value); 2680 } 2681 } 2682 2683 void Assembler::imull(Register dst, Address src) { 2684 InstructionMark im(this); 2685 prefix(src, dst, false, true /* is_map1 */); 2686 emit_int8((unsigned char)0xAF); 2687 emit_operand(dst, src, 0); 2688 } 2689 2690 void Assembler::eimull(Register dst, Register src1, Address src2, bool no_flags) { 2691 InstructionMark im(this); 2692 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, (unsigned char)0xAF, no_flags, true /* is_map1 */); 2693 } 2694 2695 void Assembler::incl(Address dst) { 2696 // Don't use it directly. Use MacroAssembler::increment() instead. 2697 InstructionMark im(this); 2698 prefix(dst); 2699 emit_int8((unsigned char)0xFF); 2700 emit_operand(rax, dst, 0); 2701 } 2702 2703 void Assembler::eincl(Register dst, Address src, bool no_flags) { 2704 // Don't use it directly. Use MacroAssembler::increment() instead. 2705 InstructionMark im(this); 2706 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2707 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 2708 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2709 emit_int8((unsigned char)0xFF); 2710 emit_operand(rax, src, 0); 2711 } 2712 2713 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) { 2714 InstructionMark im(this); 2715 assert((0 <= cc) && (cc < 16), "illegal cc"); 2716 if (L.is_bound()) { 2717 address dst = target(L); 2718 assert(dst != nullptr, "jcc most probably wrong"); 2719 2720 const int short_size = 2; 2721 const int long_size = 6; 2722 int offs = checked_cast<int>((intptr_t)dst - (intptr_t)pc()); 2723 if (maybe_short && is8bit(offs - short_size)) { 2724 // 0111 tttn #8-bit disp 2725 emit_int16(0x70 | cc, (offs - short_size) & 0xFF); 2726 } else { 2727 // 0000 1111 1000 tttn #32-bit disp 2728 assert(is_simm32(offs - long_size), 2729 "must be 32bit offset (call4)"); 2730 emit_int16(0x0F, (0x80 | cc)); 2731 emit_int32(offs - long_size); 2732 } 2733 } else { 2734 // Note: could eliminate cond. jumps to this jump if condition 2735 // is the same however, seems to be rather unlikely case. 2736 // Note: use jccb() if label to be bound is very close to get 2737 // an 8-bit displacement 2738 L.add_patch_at(code(), locator()); 2739 emit_int16(0x0F, (0x80 | cc)); 2740 emit_int32(0); 2741 } 2742 } 2743 2744 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) { 2745 if (L.is_bound()) { 2746 const int short_size = 2; 2747 address entry = target(L); 2748 #ifdef ASSERT 2749 int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size)); 2750 int delta = short_branch_delta(); 2751 if (delta != 0) { 2752 dist += (dist < 0 ? (-delta) :delta); 2753 } 2754 assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line); 2755 #endif 2756 int offs = checked_cast<int>((intptr_t)entry - (intptr_t)pc()); 2757 // 0111 tttn #8-bit disp 2758 emit_int16(0x70 | cc, (offs - short_size) & 0xFF); 2759 } else { 2760 InstructionMark im(this); 2761 L.add_patch_at(code(), locator(), file, line); 2762 emit_int16(0x70 | cc, 0); 2763 } 2764 } 2765 2766 void Assembler::jmp(Address adr) { 2767 InstructionMark im(this); 2768 prefix(adr); 2769 emit_int8((unsigned char)0xFF); 2770 emit_operand(rsp, adr, 0); 2771 } 2772 2773 void Assembler::jmp(Label& L, bool maybe_short) { 2774 if (L.is_bound()) { 2775 address entry = target(L); 2776 assert(entry != nullptr, "jmp most probably wrong"); 2777 InstructionMark im(this); 2778 const int short_size = 2; 2779 const int long_size = 5; 2780 int offs = checked_cast<int>(entry - pc()); 2781 if (maybe_short && is8bit(offs - short_size)) { 2782 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF)); 2783 } else { 2784 emit_int8((unsigned char)0xE9); 2785 emit_int32(offs - long_size); 2786 } 2787 } else { 2788 // By default, forward jumps are always 32-bit displacements, since 2789 // we can't yet know where the label will be bound. If you're sure that 2790 // the forward jump will not run beyond 256 bytes, use jmpb to 2791 // force an 8-bit displacement. 2792 InstructionMark im(this); 2793 L.add_patch_at(code(), locator()); 2794 emit_int8((unsigned char)0xE9); 2795 emit_int32(0); 2796 } 2797 } 2798 2799 void Assembler::jmp(Register entry) { 2800 int encode = prefix_and_encode(entry->encoding()); 2801 emit_int16((unsigned char)0xFF, (0xE0 | encode)); 2802 } 2803 2804 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) { 2805 InstructionMark im(this); 2806 emit_int8((unsigned char)0xE9); 2807 assert(dest != nullptr, "must have a target"); 2808 intptr_t disp = dest - (pc() + sizeof(int32_t)); 2809 assert(is_simm32(disp), "must be 32bit offset (jmp)"); 2810 emit_data(checked_cast<int32_t>(disp), rspec, call32_operand); 2811 } 2812 2813 void Assembler::jmpb_0(Label& L, const char* file, int line) { 2814 if (L.is_bound()) { 2815 const int short_size = 2; 2816 address entry = target(L); 2817 assert(entry != nullptr, "jmp most probably wrong"); 2818 #ifdef ASSERT 2819 int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size)); 2820 int delta = short_branch_delta(); 2821 if (delta != 0) { 2822 dist += (dist < 0 ? (-delta) :delta); 2823 } 2824 assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line); 2825 #endif 2826 intptr_t offs = entry - pc(); 2827 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF); 2828 } else { 2829 InstructionMark im(this); 2830 L.add_patch_at(code(), locator(), file, line); 2831 emit_int16((unsigned char)0xEB, 0); 2832 } 2833 } 2834 2835 void Assembler::ldmxcsr( Address src) { 2836 // This instruction should be SSE encoded with the REX2 prefix when an 2837 // extended GPR is present. To be consistent when UseAPX is enabled, use 2838 // this encoding even when an extended GPR is not used. 2839 if (UseAVX > 0 && !UseAPX ) { 2840 InstructionMark im(this); 2841 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2842 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2843 emit_int8((unsigned char)0xAE); 2844 emit_operand(as_Register(2), src, 0); 2845 } else { 2846 InstructionMark im(this); 2847 prefix(src, true /* is_map1 */); 2848 emit_int8((unsigned char)0xAE); 2849 emit_operand(as_Register(2), src, 0); 2850 } 2851 } 2852 2853 void Assembler::leal(Register dst, Address src) { 2854 InstructionMark im(this); 2855 prefix(src, dst); 2856 emit_int8((unsigned char)0x8D); 2857 emit_operand(dst, src, 0); 2858 } 2859 2860 void Assembler::lea(Register dst, Label& L) { 2861 emit_prefix_and_int8(get_prefixq(Address(), dst), (unsigned char)0x8D); 2862 if (!L.is_bound()) { 2863 // Patch @0x8D opcode 2864 L.add_patch_at(code(), CodeBuffer::locator(offset() - 1, sect())); 2865 // Register and [rip+disp] operand 2866 emit_modrm(0b00, raw_encode(dst), 0b101); 2867 emit_int32(0); 2868 } else { 2869 // Register and [rip+disp] operand 2870 emit_modrm(0b00, raw_encode(dst), 0b101); 2871 // Adjust displacement by sizeof lea instruction 2872 int32_t disp = checked_cast<int32_t>(target(L) - (pc() + sizeof(int32_t))); 2873 assert(is_simm32(disp), "must be 32bit offset [rip+offset]"); 2874 emit_int32(disp); 2875 } 2876 } 2877 2878 void Assembler::lfence() { 2879 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8); 2880 } 2881 2882 void Assembler::lock() { 2883 emit_int8((unsigned char)0xF0); 2884 } 2885 2886 void Assembler::size_prefix() { 2887 emit_int8(0x66); 2888 } 2889 2890 void Assembler::lzcntl(Register dst, Register src) { 2891 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2892 emit_int8((unsigned char)0xF3); 2893 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 2894 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode); 2895 } 2896 2897 void Assembler::elzcntl(Register dst, Register src, bool no_flags) { 2898 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2899 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2900 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2901 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 2902 } 2903 2904 void Assembler::lzcntl(Register dst, Address src) { 2905 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2906 InstructionMark im(this); 2907 emit_int8((unsigned char)0xF3); 2908 prefix(src, dst, false, true /* is_map1 */); 2909 emit_int8((unsigned char)0xBD); 2910 emit_operand(dst, src, 0); 2911 } 2912 2913 void Assembler::elzcntl(Register dst, Address src, bool no_flags) { 2914 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2915 InstructionMark im(this); 2916 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2917 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 2918 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 2919 emit_int8((unsigned char)0xF5); 2920 emit_operand(dst, src, 0); 2921 } 2922 2923 // Emit mfence instruction 2924 void Assembler::mfence() { 2925 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0); 2926 } 2927 2928 // Emit sfence instruction 2929 void Assembler::sfence() { 2930 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8); 2931 } 2932 2933 void Assembler::mov(Register dst, Register src) { 2934 movq(dst, src); 2935 } 2936 2937 void Assembler::movapd(XMMRegister dst, Address src) { 2938 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2939 InstructionMark im(this); 2940 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2941 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 2942 attributes.set_rex_vex_w_reverted(); 2943 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2944 emit_int8(0x28); 2945 emit_operand(dst, src, 0); 2946 } 2947 2948 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 2949 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2950 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2951 attributes.set_rex_vex_w_reverted(); 2952 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2953 emit_int16(0x28, (0xC0 | encode)); 2954 } 2955 2956 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 2957 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2958 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2959 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2960 emit_int16(0x28, (0xC0 | encode)); 2961 } 2962 2963 void Assembler::movlhps(XMMRegister dst, XMMRegister src) { 2964 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2965 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2966 emit_int16(0x16, (0xC0 | encode)); 2967 } 2968 2969 void Assembler::movb(Register dst, Address src) { 2970 InstructionMark im(this); 2971 prefix(src, dst, true); 2972 emit_int8((unsigned char)0x8A); 2973 emit_operand(dst, src, 0); 2974 } 2975 2976 void Assembler::movddup(XMMRegister dst, XMMRegister src) { 2977 assert(VM_Version::supports_sse3(), ""); 2978 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2979 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2980 attributes.set_rex_vex_w_reverted(); 2981 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2982 emit_int16(0x12, 0xC0 | encode); 2983 } 2984 2985 void Assembler::movddup(XMMRegister dst, Address src) { 2986 assert(VM_Version::supports_sse3(), ""); 2987 InstructionMark im(this); 2988 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2989 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit); 2990 attributes.set_rex_vex_w_reverted(); 2991 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2992 emit_int8(0x12); 2993 emit_operand(dst, src, 0); 2994 } 2995 2996 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) { 2997 assert(VM_Version::supports_avx(), ""); 2998 InstructionMark im(this); 2999 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3000 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit); 3001 attributes.set_rex_vex_w_reverted(); 3002 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3003 emit_int8(0x12); 3004 emit_operand(dst, src, 0); 3005 } 3006 3007 void Assembler::kmovbl(KRegister dst, KRegister src) { 3008 assert(VM_Version::supports_avx512dq(), ""); 3009 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3010 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3011 emit_int16((unsigned char)0x90, (0xC0 | encode)); 3012 } 3013 3014 void Assembler::kmovbl(KRegister dst, Register src) { 3015 assert(VM_Version::supports_avx512dq(), ""); 3016 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3017 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 3018 emit_int16((unsigned char)0x92, (0xC0 | encode)); 3019 } 3020 3021 void Assembler::kmovbl(Register dst, KRegister src) { 3022 assert(VM_Version::supports_avx512dq(), ""); 3023 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3024 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3025 emit_int16((unsigned char)0x93, (0xC0 | encode)); 3026 } 3027 3028 void Assembler::kmovwl(KRegister dst, Register src) { 3029 assert(VM_Version::supports_evex(), ""); 3030 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3031 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes, true); 3032 emit_int16((unsigned char)0x92, (0xC0 | encode)); 3033 } 3034 3035 void Assembler::kmovwl(Register dst, KRegister src) { 3036 assert(VM_Version::supports_evex(), ""); 3037 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3038 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3039 emit_int16((unsigned char)0x93, (0xC0 | encode)); 3040 } 3041 3042 void Assembler::kmovwl(KRegister dst, Address src) { 3043 assert(VM_Version::supports_evex(), ""); 3044 InstructionMark im(this); 3045 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3046 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 3047 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3048 emit_int8((unsigned char)0x90); 3049 emit_operand(dst, src, 0); 3050 } 3051 3052 void Assembler::kmovwl(Address dst, KRegister src) { 3053 assert(VM_Version::supports_evex(), ""); 3054 InstructionMark im(this); 3055 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3056 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 3057 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3058 emit_int8((unsigned char)0x91); 3059 emit_operand(src, dst, 0); 3060 } 3061 3062 void Assembler::kmovwl(KRegister dst, KRegister src) { 3063 assert(VM_Version::supports_evex(), ""); 3064 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3065 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3066 emit_int16((unsigned char)0x90, (0xC0 | encode)); 3067 } 3068 3069 void Assembler::kmovdl(KRegister dst, Register src) { 3070 assert(VM_Version::supports_avx512bw(), ""); 3071 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3072 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true); 3073 emit_int16((unsigned char)0x92, (0xC0 | encode)); 3074 } 3075 3076 void Assembler::kmovdl(Register dst, KRegister src) { 3077 assert(VM_Version::supports_avx512bw(), ""); 3078 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3079 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3080 emit_int16((unsigned char)0x93, (0xC0 | encode)); 3081 } 3082 3083 void Assembler::kmovql(KRegister dst, KRegister src) { 3084 assert(VM_Version::supports_avx512bw(), ""); 3085 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3086 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3087 emit_int16((unsigned char)0x90, (0xC0 | encode)); 3088 } 3089 3090 void Assembler::kmovql(KRegister dst, Address src) { 3091 assert(VM_Version::supports_avx512bw(), ""); 3092 InstructionMark im(this); 3093 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3094 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 3095 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3096 emit_int8((unsigned char)0x90); 3097 emit_operand(dst, src, 0); 3098 } 3099 3100 void Assembler::kmovql(Address dst, KRegister src) { 3101 assert(VM_Version::supports_avx512bw(), ""); 3102 InstructionMark im(this); 3103 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3104 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 3105 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3106 emit_int8((unsigned char)0x91); 3107 emit_operand(src, dst, 0); 3108 } 3109 3110 void Assembler::kmovql(KRegister dst, Register src) { 3111 assert(VM_Version::supports_avx512bw(), ""); 3112 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3113 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true); 3114 emit_int16((unsigned char)0x92, (0xC0 | encode)); 3115 } 3116 3117 void Assembler::kmovql(Register dst, KRegister src) { 3118 assert(VM_Version::supports_avx512bw(), ""); 3119 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3120 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3121 emit_int16((unsigned char)0x93, (0xC0 | encode)); 3122 } 3123 3124 void Assembler::knotwl(KRegister dst, KRegister src) { 3125 assert(VM_Version::supports_evex(), ""); 3126 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3127 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3128 emit_int16(0x44, (0xC0 | encode)); 3129 } 3130 3131 void Assembler::knotbl(KRegister dst, KRegister src) { 3132 assert(VM_Version::supports_evex(), ""); 3133 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3134 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3135 emit_int16(0x44, (0xC0 | encode)); 3136 } 3137 3138 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) { 3139 assert(VM_Version::supports_avx512dq(), ""); 3140 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3141 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3142 emit_int16(0x45, (0xC0 | encode)); 3143 } 3144 3145 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) { 3146 assert(VM_Version::supports_evex(), ""); 3147 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3148 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3149 emit_int16(0x45, (0xC0 | encode)); 3150 } 3151 3152 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) { 3153 assert(VM_Version::supports_avx512bw(), ""); 3154 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3155 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3156 emit_int16(0x45, (0xC0 | encode)); 3157 } 3158 3159 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) { 3160 assert(VM_Version::supports_avx512bw(), ""); 3161 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3162 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3163 emit_int16(0x45, (0xC0 | encode)); 3164 } 3165 3166 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) { 3167 assert(VM_Version::supports_avx512dq(), ""); 3168 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3169 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3170 emit_int16(0x47, (0xC0 | encode)); 3171 } 3172 3173 void Assembler::kxnorwl(KRegister dst, KRegister src1, KRegister src2) { 3174 assert(VM_Version::supports_evex(), ""); 3175 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3176 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3177 emit_int16(0x46, (0xC0 | encode)); 3178 } 3179 3180 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) { 3181 assert(VM_Version::supports_evex(), ""); 3182 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3183 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3184 emit_int16(0x47, (0xC0 | encode)); 3185 } 3186 3187 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) { 3188 assert(VM_Version::supports_avx512bw(), ""); 3189 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3190 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3191 emit_int16(0x47, (0xC0 | encode)); 3192 } 3193 3194 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) { 3195 assert(VM_Version::supports_avx512bw(), ""); 3196 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3197 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3198 emit_int16(0x47, (0xC0 | encode)); 3199 } 3200 3201 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) { 3202 assert(VM_Version::supports_avx512dq(), ""); 3203 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3204 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3205 emit_int16(0x41, (0xC0 | encode)); 3206 } 3207 3208 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) { 3209 assert(VM_Version::supports_evex(), ""); 3210 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3211 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3212 emit_int16(0x41, (0xC0 | encode)); 3213 } 3214 3215 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) { 3216 assert(VM_Version::supports_avx512bw(), ""); 3217 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3218 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3219 emit_int16(0x41, (0xC0 | encode)); 3220 } 3221 3222 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) { 3223 assert(VM_Version::supports_avx512bw(), ""); 3224 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3225 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3226 emit_int16(0x41, (0xC0 | encode)); 3227 } 3228 3229 void Assembler::knotdl(KRegister dst, KRegister src) { 3230 assert(VM_Version::supports_avx512bw(), ""); 3231 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3233 emit_int16(0x44, (0xC0 | encode)); 3234 } 3235 3236 void Assembler::knotql(KRegister dst, KRegister src) { 3237 assert(VM_Version::supports_avx512bw(), ""); 3238 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3239 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3240 emit_int16(0x44, (0xC0 | encode)); 3241 } 3242 3243 // This instruction produces ZF or CF flags 3244 void Assembler::kortestbl(KRegister src1, KRegister src2) { 3245 assert(VM_Version::supports_avx512dq(), ""); 3246 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3247 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3248 emit_int16((unsigned char)0x98, (0xC0 | encode)); 3249 } 3250 3251 // This instruction produces ZF or CF flags 3252 void Assembler::kortestwl(KRegister src1, KRegister src2) { 3253 assert(VM_Version::supports_evex(), ""); 3254 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3255 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3256 emit_int16((unsigned char)0x98, (0xC0 | encode)); 3257 } 3258 3259 // This instruction produces ZF or CF flags 3260 void Assembler::kortestdl(KRegister src1, KRegister src2) { 3261 assert(VM_Version::supports_avx512bw(), ""); 3262 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3263 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3264 emit_int16((unsigned char)0x98, (0xC0 | encode)); 3265 } 3266 3267 // This instruction produces ZF or CF flags 3268 void Assembler::kortestql(KRegister src1, KRegister src2) { 3269 assert(VM_Version::supports_avx512bw(), ""); 3270 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3271 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3272 emit_int16((unsigned char)0x98, (0xC0 | encode)); 3273 } 3274 3275 // This instruction produces ZF or CF flags 3276 void Assembler::ktestql(KRegister src1, KRegister src2) { 3277 assert(VM_Version::supports_avx512bw(), ""); 3278 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3279 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3280 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3281 } 3282 3283 void Assembler::ktestdl(KRegister src1, KRegister src2) { 3284 assert(VM_Version::supports_avx512bw(), ""); 3285 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3286 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3287 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3288 } 3289 3290 void Assembler::ktestwl(KRegister src1, KRegister src2) { 3291 assert(VM_Version::supports_avx512dq(), ""); 3292 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3293 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3294 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3295 } 3296 3297 void Assembler::ktestbl(KRegister src1, KRegister src2) { 3298 assert(VM_Version::supports_avx512dq(), ""); 3299 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3300 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3301 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3302 } 3303 3304 void Assembler::ktestq(KRegister src1, KRegister src2) { 3305 assert(VM_Version::supports_avx512bw(), ""); 3306 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3307 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3308 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3309 } 3310 3311 void Assembler::ktestd(KRegister src1, KRegister src2) { 3312 assert(VM_Version::supports_avx512bw(), ""); 3313 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3314 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3315 emit_int16((unsigned char)0x99, (0xC0 | encode)); 3316 } 3317 3318 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) { 3319 assert(VM_Version::supports_avx512dq(), ""); 3320 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3321 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3322 emit_int16(0x46, (0xC0 | encode)); 3323 } 3324 3325 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) { 3326 assert(VM_Version::supports_avx512dq(), ""); 3327 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3328 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3329 emit_int16(0x32, (0xC0 | encode)); 3330 emit_int8(imm8); 3331 } 3332 3333 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) { 3334 assert(VM_Version::supports_avx512bw(), ""); 3335 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3336 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3337 emit_int16(0x33, (0xC0 | encode)); 3338 emit_int8(imm8); 3339 } 3340 3341 3342 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) { 3343 assert(VM_Version::supports_avx512dq(), ""); 3344 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3345 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3346 emit_int16(0x30, (0xC0 | encode)); 3347 } 3348 3349 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) { 3350 assert(VM_Version::supports_evex(), ""); 3351 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3352 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3353 emit_int16(0x30, (0xC0 | encode)); 3354 emit_int8(imm8); 3355 } 3356 3357 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) { 3358 assert(VM_Version::supports_avx512bw(), ""); 3359 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3360 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3361 emit_int16(0x31, (0xC0 | encode)); 3362 emit_int8(imm8); 3363 } 3364 3365 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) { 3366 assert(VM_Version::supports_avx512bw(), ""); 3367 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3368 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3369 emit_int16(0x31, (0xC0 | encode)); 3370 emit_int8(imm8); 3371 } 3372 3373 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) { 3374 assert(VM_Version::supports_avx512bw(), ""); 3375 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3376 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3377 emit_int16(0x4B, (0xC0 | encode)); 3378 } 3379 3380 void Assembler::movb(Address dst, int imm8) { 3381 InstructionMark im(this); 3382 prefix(dst); 3383 emit_int8((unsigned char)0xC6); 3384 emit_operand(rax, dst, 1); 3385 emit_int8(imm8); 3386 } 3387 3388 3389 void Assembler::movb(Address dst, Register src) { 3390 assert(src->has_byte_register(), "must have byte register"); 3391 InstructionMark im(this); 3392 prefix(dst, src, true); 3393 emit_int8((unsigned char)0x88); 3394 emit_operand(src, dst, 0); 3395 } 3396 3397 void Assembler::movdl(XMMRegister dst, Register src) { 3398 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3399 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 3400 emit_int16(0x6E, (0xC0 | encode)); 3401 } 3402 3403 void Assembler::movdl(Register dst, XMMRegister src) { 3404 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3405 // swap src/dst to get correct prefix 3406 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 3407 emit_int16(0x7E, (0xC0 | encode)); 3408 } 3409 3410 void Assembler::movdl(XMMRegister dst, Address src) { 3411 InstructionMark im(this); 3412 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3413 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3414 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3415 emit_int8(0x6E); 3416 emit_operand(dst, src, 0); 3417 } 3418 3419 void Assembler::movdl(Address dst, XMMRegister src) { 3420 InstructionMark im(this); 3421 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3422 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3423 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3424 emit_int8(0x7E); 3425 emit_operand(src, dst, 0); 3426 } 3427 3428 void Assembler::movdqa(XMMRegister dst, XMMRegister src) { 3429 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3430 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3431 emit_int16(0x6F, (0xC0 | encode)); 3432 } 3433 3434 void Assembler::movdqa(XMMRegister dst, Address src) { 3435 InstructionMark im(this); 3436 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3437 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3438 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3439 emit_int8(0x6F); 3440 emit_operand(dst, src, 0); 3441 } 3442 3443 void Assembler::movdqu(XMMRegister dst, Address src) { 3444 InstructionMark im(this); 3445 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3446 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3447 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3448 emit_int8(0x6F); 3449 emit_operand(dst, src, 0); 3450 } 3451 3452 void Assembler::movdqu(XMMRegister dst, XMMRegister src) { 3453 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3454 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3455 emit_int16(0x6F, (0xC0 | encode)); 3456 } 3457 3458 void Assembler::movdqu(Address dst, XMMRegister src) { 3459 InstructionMark im(this); 3460 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3461 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3462 attributes.reset_is_clear_context(); 3463 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3464 emit_int8(0x7F); 3465 emit_operand(src, dst, 0); 3466 } 3467 3468 // Move Unaligned 256bit Vector 3469 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3470 assert(UseAVX > 0, ""); 3471 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3472 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3473 emit_int16(0x6F, (0xC0 | encode)); 3474 } 3475 3476 void Assembler::vmovw(XMMRegister dst, Register src) { 3477 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 3478 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3479 attributes.set_is_evex_instruction(); 3480 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes, true); 3481 emit_int16(0x6E, (0xC0 | encode)); 3482 } 3483 3484 void Assembler::vmovw(Register dst, XMMRegister src) { 3485 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 3486 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3487 attributes.set_is_evex_instruction(); 3488 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes, true); 3489 emit_int16(0x7E, (0xC0 | encode)); 3490 } 3491 3492 void Assembler::vmovdqu(XMMRegister dst, Address src) { 3493 assert(UseAVX > 0, ""); 3494 InstructionMark im(this); 3495 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3496 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3497 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3498 emit_int8(0x6F); 3499 emit_operand(dst, src, 0); 3500 } 3501 3502 void Assembler::vmovdqu(Address dst, XMMRegister src) { 3503 assert(UseAVX > 0, ""); 3504 InstructionMark im(this); 3505 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3506 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3507 attributes.reset_is_clear_context(); 3508 // swap src<->dst for encoding 3509 assert(src != xnoreg, "sanity"); 3510 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3511 emit_int8(0x7F); 3512 emit_operand(src, dst, 0); 3513 } 3514 3515 // Move Aligned 256bit Vector 3516 void Assembler::vmovdqa(XMMRegister dst, Address src) { 3517 assert(UseAVX > 0, ""); 3518 InstructionMark im(this); 3519 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3520 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3521 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3522 emit_int8(0x6F); 3523 emit_operand(dst, src, 0); 3524 } 3525 3526 void Assembler::vmovdqa(Address dst, XMMRegister src) { 3527 assert(UseAVX > 0, ""); 3528 InstructionMark im(this); 3529 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3530 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3531 attributes.reset_is_clear_context(); 3532 // swap src<->dst for encoding 3533 assert(src != xnoreg, "sanity"); 3534 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3535 emit_int8(0x7F); 3536 emit_operand(src, dst, 0); 3537 } 3538 3539 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) { 3540 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), ""); 3541 InstructionMark im(this); 3542 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false); 3543 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3544 emit_int8((unsigned char)0x8C); 3545 emit_operand(dst, src, 0); 3546 } 3547 3548 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) { 3549 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), ""); 3550 InstructionMark im(this); 3551 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false); 3552 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3553 emit_int8((unsigned char)0x8C); 3554 emit_operand(dst, src, 0); 3555 } 3556 3557 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 3558 assert(UseAVX > 0, "requires some form of AVX"); 3559 InstructionMark im(this); 3560 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3561 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3562 emit_int8(0x2C); 3563 emit_operand(dst, src, 0); 3564 } 3565 3566 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 3567 assert(UseAVX > 0, "requires some form of AVX"); 3568 InstructionMark im(this); 3569 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3570 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3571 emit_int8(0x2D); 3572 emit_operand(dst, src, 0); 3573 } 3574 3575 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) { 3576 assert(UseAVX > 0, ""); 3577 InstructionMark im(this); 3578 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3579 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3580 emit_int8(0x2E); 3581 emit_operand(src, dst, 0); 3582 } 3583 3584 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) { 3585 assert(UseAVX > 0, ""); 3586 InstructionMark im(this); 3587 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3588 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3589 emit_int8(0x2F); 3590 emit_operand(src, dst, 0); 3591 } 3592 3593 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64) 3594 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3595 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3596 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3597 attributes.set_embedded_opmask_register_specifier(mask); 3598 attributes.set_is_evex_instruction(); 3599 if (merge) { 3600 attributes.reset_is_clear_context(); 3601 } 3602 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3603 emit_int16(0x6F, (0xC0 | encode)); 3604 } 3605 3606 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { 3607 // Unmasked instruction 3608 evmovdqub(dst, k0, src, /*merge*/ false, vector_len); 3609 } 3610 3611 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3612 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3613 InstructionMark im(this); 3614 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3615 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3616 attributes.set_embedded_opmask_register_specifier(mask); 3617 attributes.set_is_evex_instruction(); 3618 if (merge) { 3619 attributes.reset_is_clear_context(); 3620 } 3621 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3622 emit_int8(0x6F); 3623 emit_operand(dst, src, 0); 3624 } 3625 3626 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) { 3627 // Unmasked instruction 3628 evmovdqub(dst, k0, src, /*merge*/ false, vector_len); 3629 } 3630 3631 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3632 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3633 assert(src != xnoreg, "sanity"); 3634 InstructionMark im(this); 3635 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3636 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3637 attributes.set_embedded_opmask_register_specifier(mask); 3638 attributes.set_is_evex_instruction(); 3639 if (merge) { 3640 attributes.reset_is_clear_context(); 3641 } 3642 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3643 emit_int8(0x7F); 3644 emit_operand(src, dst, 0); 3645 } 3646 3647 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) { 3648 // Unmasked instruction 3649 evmovdquw(dst, k0, src, /*merge*/ false, vector_len); 3650 } 3651 3652 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) { 3653 // Unmasked instruction 3654 evmovdquw(dst, k0, src, /*merge*/ false, vector_len); 3655 } 3656 3657 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3658 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3659 InstructionMark im(this); 3660 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3661 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3662 attributes.set_embedded_opmask_register_specifier(mask); 3663 attributes.set_is_evex_instruction(); 3664 if (merge) { 3665 attributes.reset_is_clear_context(); 3666 } 3667 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3668 emit_int8(0x6F); 3669 emit_operand(dst, src, 0); 3670 } 3671 3672 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) { 3673 // Unmasked instruction 3674 evmovdquw(dst, k0, src, /*merge*/ false, vector_len); 3675 } 3676 3677 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3678 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3679 assert(src != xnoreg, "sanity"); 3680 InstructionMark im(this); 3681 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3682 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3683 attributes.set_embedded_opmask_register_specifier(mask); 3684 attributes.set_is_evex_instruction(); 3685 if (merge) { 3686 attributes.reset_is_clear_context(); 3687 } 3688 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3689 emit_int8(0x7F); 3690 emit_operand(src, dst, 0); 3691 } 3692 3693 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3694 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 3695 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3696 attributes.set_embedded_opmask_register_specifier(mask); 3697 attributes.set_is_evex_instruction(); 3698 if (merge) { 3699 attributes.reset_is_clear_context(); 3700 } 3701 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3702 emit_int16(0x6F, (0xC0 | encode)); 3703 } 3704 3705 3706 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) { 3707 // Unmasked instruction 3708 evmovdqul(dst, k0, src, /*merge*/ false, vector_len); 3709 } 3710 3711 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3712 assert(VM_Version::supports_evex(), ""); 3713 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3714 attributes.set_embedded_opmask_register_specifier(mask); 3715 attributes.set_is_evex_instruction(); 3716 if (merge) { 3717 attributes.reset_is_clear_context(); 3718 } 3719 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3720 emit_int16(0x6F, (0xC0 | encode)); 3721 } 3722 3723 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) { 3724 // Unmasked instruction 3725 evmovdqul(dst, k0, src, /*merge*/ false, vector_len); 3726 } 3727 3728 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3729 assert(VM_Version::supports_evex(), ""); 3730 InstructionMark im(this); 3731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true); 3732 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3733 attributes.set_embedded_opmask_register_specifier(mask); 3734 attributes.set_is_evex_instruction(); 3735 if (merge) { 3736 attributes.reset_is_clear_context(); 3737 } 3738 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3739 emit_int8(0x6F); 3740 emit_operand(dst, src, 0); 3741 } 3742 3743 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) { 3744 // Unmasked isntruction 3745 evmovdqul(dst, k0, src, /*merge*/ true, vector_len); 3746 } 3747 3748 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3749 assert(VM_Version::supports_evex(), ""); 3750 assert(src != xnoreg, "sanity"); 3751 InstructionMark im(this); 3752 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3753 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3754 attributes.set_embedded_opmask_register_specifier(mask); 3755 attributes.set_is_evex_instruction(); 3756 if (merge) { 3757 attributes.reset_is_clear_context(); 3758 } 3759 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3760 emit_int8(0x7F); 3761 emit_operand(src, dst, 0); 3762 } 3763 3764 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) { 3765 // Unmasked instruction 3766 evmovdquq(dst, k0, src, /*merge*/ false, vector_len); 3767 } 3768 3769 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3770 assert(VM_Version::supports_evex(), ""); 3771 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3772 attributes.set_embedded_opmask_register_specifier(mask); 3773 attributes.set_is_evex_instruction(); 3774 if (merge) { 3775 attributes.reset_is_clear_context(); 3776 } 3777 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3778 emit_int16(0x6F, (0xC0 | encode)); 3779 } 3780 3781 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) { 3782 // Unmasked instruction 3783 evmovdquq(dst, k0, src, /*merge*/ false, vector_len); 3784 } 3785 3786 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3787 assert(VM_Version::supports_evex(), ""); 3788 InstructionMark im(this); 3789 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3790 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3791 attributes.set_embedded_opmask_register_specifier(mask); 3792 attributes.set_is_evex_instruction(); 3793 if (merge) { 3794 attributes.reset_is_clear_context(); 3795 } 3796 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3797 emit_int8(0x6F); 3798 emit_operand(dst, src, 0); 3799 } 3800 3801 // Move Aligned 512bit Vector 3802 void Assembler::evmovdqaq(XMMRegister dst, Address src, int vector_len) { 3803 // Unmasked instruction 3804 evmovdqaq(dst, k0, src, /*merge*/ false, vector_len); 3805 } 3806 3807 void Assembler::evmovdqaq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3808 assert(VM_Version::supports_evex(), ""); 3809 InstructionMark im(this); 3810 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3811 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3812 attributes.set_embedded_opmask_register_specifier(mask); 3813 attributes.set_is_evex_instruction(); 3814 if (merge) { 3815 attributes.reset_is_clear_context(); 3816 } 3817 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3818 emit_int8(0x6F); 3819 emit_operand(dst, src, 0); 3820 } 3821 3822 void Assembler::evmovntdquq(Address dst, XMMRegister src, int vector_len) { 3823 // Unmasked instruction 3824 evmovntdquq(dst, k0, src, /*merge*/ true, vector_len); 3825 } 3826 3827 void Assembler::evmovntdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3828 assert(VM_Version::supports_evex(), ""); 3829 assert(src != xnoreg, "sanity"); 3830 InstructionMark im(this); 3831 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3832 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3833 attributes.set_embedded_opmask_register_specifier(mask); 3834 if (merge) { 3835 attributes.reset_is_clear_context(); 3836 } 3837 attributes.set_is_evex_instruction(); 3838 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3839 emit_int8(0xE7); 3840 emit_operand(src, dst, 0); 3841 } 3842 3843 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) { 3844 // Unmasked instruction 3845 evmovdquq(dst, k0, src, /*merge*/ true, vector_len); 3846 } 3847 3848 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3849 assert(VM_Version::supports_evex(), ""); 3850 assert(src != xnoreg, "sanity"); 3851 InstructionMark im(this); 3852 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3853 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3854 attributes.set_embedded_opmask_register_specifier(mask); 3855 if (merge) { 3856 attributes.reset_is_clear_context(); 3857 } 3858 attributes.set_is_evex_instruction(); 3859 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3860 emit_int8(0x7F); 3861 emit_operand(src, dst, 0); 3862 } 3863 3864 // Uses zero extension on 64bit 3865 3866 void Assembler::movl(Register dst, int32_t imm32) { 3867 int encode = prefix_and_encode(dst->encoding()); 3868 emit_int8(0xB8 | encode); 3869 emit_int32(imm32); 3870 } 3871 3872 void Assembler::movl(Register dst, Register src) { 3873 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 3874 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 3875 } 3876 3877 void Assembler::movl(Register dst, Address src) { 3878 InstructionMark im(this); 3879 prefix(src, dst); 3880 emit_int8((unsigned char)0x8B); 3881 emit_operand(dst, src, 0); 3882 } 3883 3884 void Assembler::movl(Address dst, int32_t imm32) { 3885 InstructionMark im(this); 3886 prefix(dst); 3887 emit_int8((unsigned char)0xC7); 3888 emit_operand(rax, dst, 4); 3889 emit_int32(imm32); 3890 } 3891 3892 void Assembler::movl(Address dst, Register src) { 3893 InstructionMark im(this); 3894 prefix(dst, src); 3895 emit_int8((unsigned char)0x89); 3896 emit_operand(src, dst, 0); 3897 } 3898 3899 // New cpus require to use movsd and movss to avoid partial register stall 3900 // when loading from memory. But for old Opteron use movlpd instead of movsd. 3901 // The selection is done in MacroAssembler::movdbl() and movflt(). 3902 void Assembler::movlpd(XMMRegister dst, Address src) { 3903 InstructionMark im(this); 3904 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3905 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3906 attributes.set_rex_vex_w_reverted(); 3907 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3908 emit_int8(0x12); 3909 emit_operand(dst, src, 0); 3910 } 3911 3912 void Assembler::movq(XMMRegister dst, Address src) { 3913 InstructionMark im(this); 3914 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3915 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3916 attributes.set_rex_vex_w_reverted(); 3917 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3918 emit_int8(0x7E); 3919 emit_operand(dst, src, 0); 3920 } 3921 3922 void Assembler::movq(Address dst, XMMRegister src) { 3923 InstructionMark im(this); 3924 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3925 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3926 attributes.set_rex_vex_w_reverted(); 3927 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3928 emit_int8((unsigned char)0xD6); 3929 emit_operand(src, dst, 0); 3930 } 3931 3932 void Assembler::movq(XMMRegister dst, XMMRegister src) { 3933 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3934 attributes.set_rex_vex_w_reverted(); 3935 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3936 emit_int16((unsigned char)0xD6, (0xC0 | encode)); 3937 } 3938 3939 void Assembler::movq(Register dst, XMMRegister src) { 3940 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3941 // swap src/dst to get correct prefix 3942 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 3943 emit_int16(0x7E, (0xC0 | encode)); 3944 } 3945 3946 void Assembler::movq(XMMRegister dst, Register src) { 3947 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3948 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 3949 emit_int16(0x6E, (0xC0 | encode)); 3950 } 3951 3952 void Assembler::movsbl(Register dst, Address src) { // movsxb 3953 InstructionMark im(this); 3954 prefix(src, dst, false, true /* is_map1 */); 3955 emit_int8((unsigned char)0xBE); 3956 emit_operand(dst, src, 0); 3957 } 3958 3959 void Assembler::movsbl(Register dst, Register src) { // movsxb 3960 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true, true /* is_map1 */); 3961 emit_opcode_prefix_and_encoding((unsigned char)0xBE, 0xC0, encode); 3962 } 3963 3964 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 3965 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3966 attributes.set_rex_vex_w_reverted(); 3967 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3968 emit_int16(0x10, (0xC0 | encode)); 3969 } 3970 3971 void Assembler::movsd(XMMRegister dst, Address src) { 3972 InstructionMark im(this); 3973 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3974 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3975 attributes.set_rex_vex_w_reverted(); 3976 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3977 emit_int8(0x10); 3978 emit_operand(dst, src, 0); 3979 } 3980 3981 void Assembler::movsd(Address dst, XMMRegister src) { 3982 InstructionMark im(this); 3983 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3984 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3985 attributes.reset_is_clear_context(); 3986 attributes.set_rex_vex_w_reverted(); 3987 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3988 emit_int8(0x11); 3989 emit_operand(src, dst, 0); 3990 } 3991 3992 void Assembler::vmovsd(XMMRegister dst, XMMRegister src, XMMRegister src2) { 3993 assert(UseAVX > 0, "Requires some form of AVX"); 3994 InstructionMark im(this); 3995 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3996 int encode = vex_prefix_and_encode(src2->encoding(), src->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3997 emit_int16(0x11, (0xC0 | encode)); 3998 } 3999 4000 void Assembler::movss(XMMRegister dst, XMMRegister src) { 4001 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4002 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 4003 emit_int16(0x10, (0xC0 | encode)); 4004 } 4005 4006 void Assembler::movss(XMMRegister dst, Address src) { 4007 InstructionMark im(this); 4008 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 4010 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 4011 emit_int8(0x10); 4012 emit_operand(dst, src, 0); 4013 } 4014 4015 void Assembler::movss(Address dst, XMMRegister src) { 4016 InstructionMark im(this); 4017 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4018 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 4019 attributes.reset_is_clear_context(); 4020 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 4021 emit_int8(0x11); 4022 emit_operand(src, dst, 0); 4023 } 4024 4025 void Assembler::movswl(Register dst, Address src) { // movsxw 4026 InstructionMark im(this); 4027 prefix(src, dst, false, true /* is_map1 */); 4028 emit_int8((unsigned char)0xBF); 4029 emit_operand(dst, src, 0); 4030 } 4031 4032 void Assembler::movswl(Register dst, Register src) { // movsxw 4033 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 4034 emit_opcode_prefix_and_encoding((unsigned char)0xBF, 0xC0, encode); 4035 } 4036 4037 void Assembler::movups(XMMRegister dst, Address src) { 4038 InstructionMark im(this); 4039 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4040 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4041 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 4042 emit_int8(0x10); 4043 emit_operand(dst, src, 0); 4044 } 4045 4046 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) { 4047 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 4048 InstructionMark im(this); 4049 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4050 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4051 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 4052 emit_int8(0x10); 4053 emit_operand(dst, src, 0); 4054 } 4055 4056 void Assembler::movups(Address dst, XMMRegister src) { 4057 InstructionMark im(this); 4058 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4059 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4060 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 4061 emit_int8(0x11); 4062 emit_operand(src, dst, 0); 4063 } 4064 4065 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) { 4066 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 4067 InstructionMark im(this); 4068 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4069 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4070 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 4071 emit_int8(0x11); 4072 emit_operand(src, dst, 0); 4073 } 4074 4075 void Assembler::movw(Address dst, int imm16) { 4076 InstructionMark im(this); 4077 4078 emit_int8(0x66); // switch to 16-bit mode 4079 prefix(dst); 4080 emit_int8((unsigned char)0xC7); 4081 emit_operand(rax, dst, 2); 4082 emit_int16(imm16); 4083 } 4084 4085 void Assembler::movw(Register dst, Address src) { 4086 InstructionMark im(this); 4087 emit_int8(0x66); 4088 prefix(src, dst); 4089 emit_int8((unsigned char)0x8B); 4090 emit_operand(dst, src, 0); 4091 } 4092 4093 void Assembler::movw(Address dst, Register src) { 4094 InstructionMark im(this); 4095 emit_int8(0x66); 4096 prefix(dst, src); 4097 emit_int8((unsigned char)0x89); 4098 emit_operand(src, dst, 0); 4099 } 4100 4101 void Assembler::movzbl(Register dst, Address src) { // movzxb 4102 InstructionMark im(this); 4103 prefix(src, dst, false, true /* is_map1 */); 4104 emit_int8((unsigned char)0xB6); 4105 emit_operand(dst, src, 0); 4106 } 4107 4108 void Assembler::movzbl(Register dst, Register src) { // movzxb 4109 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true, true /* is_map1 */); 4110 emit_opcode_prefix_and_encoding((unsigned char)0xB6, 0xC0, encode); 4111 } 4112 4113 void Assembler::movzwl(Register dst, Address src) { // movzxw 4114 InstructionMark im(this); 4115 prefix(src, dst, false, true /* is_map1 */); 4116 emit_int8((unsigned char)0xB7); 4117 emit_operand(dst, src, 0); 4118 } 4119 4120 void Assembler::movzwl(Register dst, Register src) { // movzxw 4121 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 4122 emit_opcode_prefix_and_encoding((unsigned char)0xB7, 0xC0, encode); 4123 } 4124 4125 void Assembler::mull(Address src) { 4126 InstructionMark im(this); 4127 prefix(src); 4128 emit_int8((unsigned char)0xF7); 4129 emit_operand(rsp, src, 0); 4130 } 4131 4132 void Assembler::emull(Address src, bool no_flags) { 4133 InstructionMark im(this); 4134 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4135 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 4136 eevex_prefix_nf(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4137 emit_int8((unsigned char)0xF7); 4138 emit_operand(rsp, src, 0); 4139 } 4140 4141 void Assembler::mull(Register src) { 4142 int encode = prefix_and_encode(src->encoding()); 4143 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 4144 } 4145 4146 void Assembler::emull(Register src, bool no_flags) { 4147 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4148 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4149 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 4150 } 4151 4152 void Assembler::mulsd(XMMRegister dst, Address src) { 4153 InstructionMark im(this); 4154 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4155 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 4156 attributes.set_rex_vex_w_reverted(); 4157 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 4158 emit_int8(0x59); 4159 emit_operand(dst, src, 0); 4160 } 4161 4162 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 4163 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4164 attributes.set_rex_vex_w_reverted(); 4165 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 4166 emit_int16(0x59, (0xC0 | encode)); 4167 } 4168 4169 void Assembler::mulss(XMMRegister dst, Address src) { 4170 InstructionMark im(this); 4171 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4172 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 4173 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 4174 emit_int8(0x59); 4175 emit_operand(dst, src, 0); 4176 } 4177 4178 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 4179 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4180 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 4181 emit_int16(0x59, (0xC0 | encode)); 4182 } 4183 4184 void Assembler::negl(Register dst) { 4185 int encode = prefix_and_encode(dst->encoding()); 4186 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 4187 } 4188 4189 void Assembler::enegl(Register dst, Register src, bool no_flags) { 4190 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4191 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4192 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 4193 } 4194 4195 void Assembler::negl(Address dst) { 4196 InstructionMark im(this); 4197 prefix(dst); 4198 emit_int8((unsigned char)0xF7); 4199 emit_operand(as_Register(3), dst, 0); 4200 } 4201 4202 void Assembler::enegl(Register dst, Address src, bool no_flags) { 4203 InstructionMark im(this); 4204 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4205 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 4206 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4207 emit_int8((unsigned char)0xF7); 4208 emit_operand(as_Register(3), src, 0); 4209 } 4210 4211 void Assembler::nop(uint i) { 4212 #ifdef ASSERT 4213 assert(i > 0, " "); 4214 // The fancy nops aren't currently recognized by debuggers making it a 4215 // pain to disassemble code while debugging. If asserts are on clearly 4216 // speed is not an issue so simply use the single byte traditional nop 4217 // to do alignment. 4218 4219 for (; i > 0 ; i--) emit_int8((unsigned char)0x90); 4220 return; 4221 4222 #endif // ASSERT 4223 4224 if (UseAddressNop && VM_Version::is_intel()) { 4225 // 4226 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel 4227 // 1: 0x90 4228 // 2: 0x66 0x90 4229 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 4230 // 4: 0x0F 0x1F 0x40 0x00 4231 // 5: 0x0F 0x1F 0x44 0x00 0x00 4232 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 4233 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 4234 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4235 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4236 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4237 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4238 4239 // The rest coding is Intel specific - don't use consecutive address nops 4240 4241 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4242 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4243 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4244 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4245 4246 while(i >= 15) { 4247 // For Intel don't generate consecutive address nops (mix with regular nops) 4248 i -= 15; 4249 emit_int24(0x66, 0x66, 0x66); 4250 addr_nop_8(); 4251 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4252 } 4253 switch (i) { 4254 case 14: 4255 emit_int8(0x66); // size prefix 4256 case 13: 4257 emit_int8(0x66); // size prefix 4258 case 12: 4259 addr_nop_8(); 4260 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4261 break; 4262 case 11: 4263 emit_int8(0x66); // size prefix 4264 case 10: 4265 emit_int8(0x66); // size prefix 4266 case 9: 4267 emit_int8(0x66); // size prefix 4268 case 8: 4269 addr_nop_8(); 4270 break; 4271 case 7: 4272 addr_nop_7(); 4273 break; 4274 case 6: 4275 emit_int8(0x66); // size prefix 4276 case 5: 4277 addr_nop_5(); 4278 break; 4279 case 4: 4280 addr_nop_4(); 4281 break; 4282 case 3: 4283 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 4284 emit_int8(0x66); // size prefix 4285 case 2: 4286 emit_int8(0x66); // size prefix 4287 case 1: 4288 emit_int8((unsigned char)0x90); 4289 // nop 4290 break; 4291 default: 4292 assert(i == 0, " "); 4293 } 4294 return; 4295 } 4296 if (UseAddressNop && VM_Version::is_amd_family()) { 4297 // 4298 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD. 4299 // 1: 0x90 4300 // 2: 0x66 0x90 4301 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 4302 // 4: 0x0F 0x1F 0x40 0x00 4303 // 5: 0x0F 0x1F 0x44 0x00 0x00 4304 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 4305 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 4306 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4307 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4308 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4309 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4310 4311 // The rest coding is AMD specific - use consecutive address nops 4312 4313 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00 4314 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00 4315 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 4316 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 4317 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4318 // Size prefixes (0x66) are added for larger sizes 4319 4320 while(i >= 22) { 4321 i -= 11; 4322 emit_int24(0x66, 0x66, 0x66); 4323 addr_nop_8(); 4324 } 4325 // Generate first nop for size between 21-12 4326 switch (i) { 4327 case 21: 4328 i -= 1; 4329 emit_int8(0x66); // size prefix 4330 case 20: 4331 case 19: 4332 i -= 1; 4333 emit_int8(0x66); // size prefix 4334 case 18: 4335 case 17: 4336 i -= 1; 4337 emit_int8(0x66); // size prefix 4338 case 16: 4339 case 15: 4340 i -= 8; 4341 addr_nop_8(); 4342 break; 4343 case 14: 4344 case 13: 4345 i -= 7; 4346 addr_nop_7(); 4347 break; 4348 case 12: 4349 i -= 6; 4350 emit_int8(0x66); // size prefix 4351 addr_nop_5(); 4352 break; 4353 default: 4354 assert(i < 12, " "); 4355 } 4356 4357 // Generate second nop for size between 11-1 4358 switch (i) { 4359 case 11: 4360 emit_int8(0x66); // size prefix 4361 case 10: 4362 emit_int8(0x66); // size prefix 4363 case 9: 4364 emit_int8(0x66); // size prefix 4365 case 8: 4366 addr_nop_8(); 4367 break; 4368 case 7: 4369 addr_nop_7(); 4370 break; 4371 case 6: 4372 emit_int8(0x66); // size prefix 4373 case 5: 4374 addr_nop_5(); 4375 break; 4376 case 4: 4377 addr_nop_4(); 4378 break; 4379 case 3: 4380 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 4381 emit_int8(0x66); // size prefix 4382 case 2: 4383 emit_int8(0x66); // size prefix 4384 case 1: 4385 emit_int8((unsigned char)0x90); 4386 // nop 4387 break; 4388 default: 4389 assert(i == 0, " "); 4390 } 4391 return; 4392 } 4393 4394 if (UseAddressNop && VM_Version::is_zx()) { 4395 // 4396 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX 4397 // 1: 0x90 4398 // 2: 0x66 0x90 4399 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 4400 // 4: 0x0F 0x1F 0x40 0x00 4401 // 5: 0x0F 0x1F 0x44 0x00 0x00 4402 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 4403 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 4404 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4405 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4406 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4407 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 4408 4409 // The rest coding is ZX specific - don't use consecutive address nops 4410 4411 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4412 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4413 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4414 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 4415 4416 while (i >= 15) { 4417 // For ZX don't generate consecutive address nops (mix with regular nops) 4418 i -= 15; 4419 emit_int24(0x66, 0x66, 0x66); 4420 addr_nop_8(); 4421 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4422 } 4423 switch (i) { 4424 case 14: 4425 emit_int8(0x66); // size prefix 4426 case 13: 4427 emit_int8(0x66); // size prefix 4428 case 12: 4429 addr_nop_8(); 4430 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4431 break; 4432 case 11: 4433 emit_int8(0x66); // size prefix 4434 case 10: 4435 emit_int8(0x66); // size prefix 4436 case 9: 4437 emit_int8(0x66); // size prefix 4438 case 8: 4439 addr_nop_8(); 4440 break; 4441 case 7: 4442 addr_nop_7(); 4443 break; 4444 case 6: 4445 emit_int8(0x66); // size prefix 4446 case 5: 4447 addr_nop_5(); 4448 break; 4449 case 4: 4450 addr_nop_4(); 4451 break; 4452 case 3: 4453 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 4454 emit_int8(0x66); // size prefix 4455 case 2: 4456 emit_int8(0x66); // size prefix 4457 case 1: 4458 emit_int8((unsigned char)0x90); 4459 // nop 4460 break; 4461 default: 4462 assert(i == 0, " "); 4463 } 4464 return; 4465 } 4466 4467 // Using nops with size prefixes "0x66 0x90". 4468 // From AMD Optimization Guide: 4469 // 1: 0x90 4470 // 2: 0x66 0x90 4471 // 3: 0x66 0x66 0x90 4472 // 4: 0x66 0x66 0x66 0x90 4473 // 5: 0x66 0x66 0x90 0x66 0x90 4474 // 6: 0x66 0x66 0x90 0x66 0x66 0x90 4475 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 4476 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90 4477 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90 4478 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90 4479 // 4480 while (i > 12) { 4481 i -= 4; 4482 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4483 } 4484 // 1 - 12 nops 4485 if (i > 8) { 4486 if (i > 9) { 4487 i -= 1; 4488 emit_int8(0x66); 4489 } 4490 i -= 3; 4491 emit_int24(0x66, 0x66, (unsigned char)0x90); 4492 } 4493 // 1 - 8 nops 4494 if (i > 4) { 4495 if (i > 6) { 4496 i -= 1; 4497 emit_int8(0x66); 4498 } 4499 i -= 3; 4500 emit_int24(0x66, 0x66, (unsigned char)0x90); 4501 } 4502 switch (i) { 4503 case 4: 4504 emit_int8(0x66); 4505 case 3: 4506 emit_int8(0x66); 4507 case 2: 4508 emit_int8(0x66); 4509 case 1: 4510 emit_int8((unsigned char)0x90); 4511 break; 4512 default: 4513 assert(i == 0, " "); 4514 } 4515 } 4516 4517 void Assembler::notl(Register dst) { 4518 int encode = prefix_and_encode(dst->encoding()); 4519 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 4520 } 4521 4522 void Assembler::enotl(Register dst, Register src) { 4523 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4524 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes); 4525 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 4526 } 4527 4528 void Assembler::eorw(Register dst, Register src1, Register src2, bool no_flags) { 4529 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x0B, 0xC0, no_flags, true /* is_commutative */); 4530 } 4531 4532 void Assembler::orl(Address dst, int32_t imm32) { 4533 InstructionMark im(this); 4534 prefix(dst); 4535 emit_arith_operand(0x81, rcx, dst, imm32); 4536 } 4537 4538 void Assembler::eorl(Register dst, Address src, int32_t imm32, bool no_flags) { 4539 InstructionMark im(this); 4540 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4541 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 4542 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4543 emit_arith_operand(0x81, rcx, src, imm32); 4544 } 4545 4546 void Assembler::orl(Register dst, int32_t imm32) { 4547 prefix(dst); 4548 emit_arith(0x81, 0xC8, dst, imm32); 4549 } 4550 4551 void Assembler::eorl(Register dst, Register src, int32_t imm32, bool no_flags) { 4552 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xC8, no_flags); 4553 } 4554 4555 void Assembler::orl(Register dst, Address src) { 4556 InstructionMark im(this); 4557 prefix(src, dst); 4558 emit_int8(0x0B); 4559 emit_operand(dst, src, 0); 4560 } 4561 4562 void Assembler::eorl(Register dst, Register src1, Address src2, bool no_flags) { 4563 InstructionMark im(this); 4564 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, no_flags); 4565 } 4566 4567 void Assembler::orl(Register dst, Register src) { 4568 (void) prefix_and_encode(dst->encoding(), src->encoding()); 4569 emit_arith(0x0B, 0xC0, dst, src); 4570 } 4571 4572 void Assembler::eorl(Register dst, Register src1, Register src2, bool no_flags) { 4573 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, 0xC0, no_flags, true /* is_commutative */); 4574 } 4575 4576 void Assembler::orl(Address dst, Register src) { 4577 InstructionMark im(this); 4578 prefix(dst, src); 4579 emit_int8(0x09); 4580 emit_operand(src, dst, 0); 4581 } 4582 4583 void Assembler::eorl(Register dst, Address src1, Register src2, bool no_flags) { 4584 InstructionMark im(this); 4585 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */); 4586 } 4587 4588 void Assembler::orb(Address dst, int imm8) { 4589 InstructionMark im(this); 4590 prefix(dst); 4591 emit_int8((unsigned char)0x80); 4592 emit_operand(rcx, dst, 1); 4593 emit_int8(imm8); 4594 } 4595 4596 void Assembler::eorb(Register dst, Address src, int imm8, bool no_flags) { 4597 InstructionMark im(this); 4598 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4599 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit); 4600 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 4601 emit_int8((unsigned char)0x80); 4602 emit_operand(rcx, src, 1); 4603 emit_int8(imm8); 4604 } 4605 4606 void Assembler::orb(Address dst, Register src) { 4607 InstructionMark im(this); 4608 prefix(dst, src, true); 4609 emit_int8(0x08); 4610 emit_operand(src, dst, 0); 4611 } 4612 4613 void Assembler::eorb(Register dst, Address src1, Register src2, bool no_flags) { 4614 InstructionMark im(this); 4615 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x08, no_flags, false /* is_map1 */, true /* is_commutative */); 4616 } 4617 4618 void Assembler::packsswb(XMMRegister dst, XMMRegister src) { 4619 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4620 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4621 emit_int16(0x63, (0xC0 | encode)); 4622 } 4623 4624 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4625 assert(UseAVX > 0, "some form of AVX must be enabled"); 4626 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4627 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4628 emit_int16(0x63, (0xC0 | encode)); 4629 } 4630 4631 void Assembler::packssdw(XMMRegister dst, XMMRegister src) { 4632 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4633 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4634 emit_int16(0x6B, (0xC0 | encode)); 4635 } 4636 4637 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4638 assert(UseAVX > 0, "some form of AVX must be enabled"); 4639 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4640 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4641 emit_int16(0x6B, (0xC0 | encode)); 4642 } 4643 4644 void Assembler::packuswb(XMMRegister dst, Address src) { 4645 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 4646 InstructionMark im(this); 4647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4648 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4649 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4650 emit_int8(0x67); 4651 emit_operand(dst, src, 0); 4652 } 4653 4654 void Assembler::packuswb(XMMRegister dst, XMMRegister src) { 4655 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4656 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4657 emit_int16(0x67, (0xC0 | encode)); 4658 } 4659 4660 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4661 assert(UseAVX > 0, "some form of AVX must be enabled"); 4662 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4663 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4664 emit_int16(0x67, (0xC0 | encode)); 4665 } 4666 4667 void Assembler::packusdw(XMMRegister dst, XMMRegister src) { 4668 assert(VM_Version::supports_sse4_1(), ""); 4669 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4670 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4671 emit_int16(0x2B, (0xC0 | encode)); 4672 } 4673 4674 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4675 assert(UseAVX > 0, "some form of AVX must be enabled"); 4676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4677 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4678 emit_int16(0x2B, (0xC0 | encode)); 4679 } 4680 4681 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4682 assert(VM_Version::supports_avx2(), ""); 4683 assert(vector_len != AVX_128bit, ""); 4684 // VEX.256.66.0F3A.W1 00 /r ib 4685 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4686 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4687 emit_int24(0x00, (0xC0 | encode), imm8); 4688 } 4689 4690 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4691 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() : 4692 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported"); 4693 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4694 attributes.set_is_evex_instruction(); 4695 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4696 emit_int16(0x36, (0xC0 | encode)); 4697 } 4698 4699 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4700 assert(VM_Version::supports_avx512_vbmi(), ""); 4701 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4702 attributes.set_is_evex_instruction(); 4703 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4704 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 4705 } 4706 4707 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 4708 assert(VM_Version::supports_avx512_vbmi(), ""); 4709 InstructionMark im(this); 4710 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4711 attributes.set_is_evex_instruction(); 4712 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4713 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4714 emit_int8((unsigned char)0x8D); 4715 emit_operand(dst, src, 0); 4716 } 4717 4718 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4719 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() : 4720 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() : 4721 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported"); 4722 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4723 attributes.set_is_evex_instruction(); 4724 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4725 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 4726 } 4727 4728 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4729 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) || 4730 (vector_len == AVX_512bit && VM_Version::supports_evex()), ""); 4731 // VEX.NDS.256.66.0F38.W0 36 /r 4732 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4733 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4734 emit_int16(0x36, (0xC0 | encode)); 4735 } 4736 4737 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 4738 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) || 4739 (vector_len == AVX_512bit && VM_Version::supports_evex()), ""); 4740 // VEX.NDS.256.66.0F38.W0 36 /r 4741 InstructionMark im(this); 4742 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 4744 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4745 emit_int8(0x36); 4746 emit_operand(dst, src, 0); 4747 } 4748 4749 void Assembler::vpermps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4750 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) || 4751 (vector_len == AVX_512bit && VM_Version::supports_evex()), ""); 4752 // VEX.NDS.XXX.66.0F38.W0 16 /r 4753 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4754 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4755 emit_int16(0x16, (0xC0 | encode)); 4756 } 4757 4758 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 4759 assert(VM_Version::supports_avx2(), ""); 4760 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4761 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4762 emit_int24(0x46, (0xC0 | encode), imm8); 4763 } 4764 4765 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 4766 assert(VM_Version::supports_avx(), ""); 4767 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4769 emit_int24(0x06, (0xC0 | encode), imm8); 4770 } 4771 4772 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4773 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 4774 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4775 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4776 emit_int24(0x04, (0xC0 | encode), imm8); 4777 } 4778 4779 void Assembler::vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4780 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 4781 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4782 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4783 emit_int16(0x0C, (0xC0 | encode)); 4784 } 4785 4786 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4787 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 4788 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false); 4789 attributes.set_rex_vex_w_reverted(); 4790 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4791 emit_int24(0x05, (0xC0 | encode), imm8); 4792 } 4793 4794 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4795 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), ""); 4796 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false); 4797 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4798 emit_int24(0x01, (0xC0 | encode), imm8); 4799 } 4800 4801 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) { 4802 assert(VM_Version::supports_avx512_vbmi(), ""); 4803 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4804 attributes.set_is_evex_instruction(); 4805 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4806 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode)); 4807 } 4808 4809 void Assembler::pause() { 4810 emit_int16((unsigned char)0xF3, (unsigned char)0x90); 4811 } 4812 4813 void Assembler::ud2() { 4814 emit_int16(0x0F, 0x0B); 4815 } 4816 4817 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 4818 assert(VM_Version::supports_sse4_2(), ""); 4819 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs as BASE or INDEX of address operand"); 4820 InstructionMark im(this); 4821 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4822 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4823 emit_int8(0x61); 4824 emit_operand(dst, src, 1); 4825 emit_int8(imm8); 4826 } 4827 4828 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 4829 assert(VM_Version::supports_sse4_2(), ""); 4830 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4831 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4832 emit_int24(0x61, (0xC0 | encode), imm8); 4833 } 4834 4835 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4836 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 4837 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4838 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4839 emit_int16(0x74, (0xC0 | encode)); 4840 } 4841 4842 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) { 4843 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4844 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4845 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4846 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4847 emit_int16(cond_encoding, (0xC0 | encode)); 4848 } 4849 4850 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4851 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4852 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4853 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4854 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4855 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4856 emit_int16(0x74, (0xC0 | encode)); 4857 } 4858 4859 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 4860 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4861 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4862 InstructionMark im(this); 4863 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4864 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4865 emit_int8(0x74); 4866 emit_operand(dst, src2, 0); 4867 } 4868 4869 // In this context, kdst is written the mask used to process the equal components 4870 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 4871 assert(VM_Version::supports_avx512bw(), ""); 4872 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4873 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4874 attributes.set_is_evex_instruction(); 4875 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4876 emit_int16(0x74, (0xC0 | encode)); 4877 } 4878 4879 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4880 assert(VM_Version::supports_avx512bw(), ""); 4881 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4882 InstructionMark im(this); 4883 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4884 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4885 attributes.set_is_evex_instruction(); 4886 int dst_enc = kdst->encoding(); 4887 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4888 emit_int8(0x64); 4889 emit_operand(as_Register(dst_enc), src, 0); 4890 } 4891 4892 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 4893 assert(VM_Version::supports_avx512bw(), ""); 4894 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4895 InstructionMark im(this); 4896 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4897 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4898 attributes.reset_is_clear_context(); 4899 attributes.set_embedded_opmask_register_specifier(mask); 4900 attributes.set_is_evex_instruction(); 4901 int dst_enc = kdst->encoding(); 4902 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4903 emit_int8(0x64); 4904 emit_operand(as_Register(dst_enc), src, 0); 4905 } 4906 4907 void Assembler::evpcmpub(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { 4908 assert(VM_Version::supports_avx512bw(), ""); 4909 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4910 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4911 attributes.set_is_evex_instruction(); 4912 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4913 emit_int24(0x3E, (0xC0 | encode), vcc); 4914 } 4915 4916 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { 4917 assert(VM_Version::supports_avx512bw(), ""); 4918 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4919 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4920 attributes.set_is_evex_instruction(); 4921 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4922 emit_int24(0x3E, (0xC0 | encode), vcc); 4923 } 4924 4925 void Assembler::evpcmpud(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { 4926 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4927 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4928 attributes.set_is_evex_instruction(); 4929 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4930 emit_int24(0x1E, (0xC0 | encode), vcc); 4931 } 4932 4933 void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { 4934 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4935 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4936 attributes.set_is_evex_instruction(); 4937 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4938 emit_int24(0x1E, (0xC0 | encode), vcc); 4939 } 4940 4941 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) { 4942 assert(VM_Version::supports_avx512bw(), ""); 4943 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4944 InstructionMark im(this); 4945 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4946 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4947 attributes.set_is_evex_instruction(); 4948 int dst_enc = kdst->encoding(); 4949 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4950 emit_int8(0x3E); 4951 emit_operand(as_Register(dst_enc), src, 1); 4952 emit_int8(vcc); 4953 } 4954 4955 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4956 assert(VM_Version::supports_avx512bw(), ""); 4957 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4958 InstructionMark im(this); 4959 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4960 attributes.set_is_evex_instruction(); 4961 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4962 int dst_enc = kdst->encoding(); 4963 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4964 emit_int8(0x74); 4965 emit_operand(as_Register(dst_enc), src, 0); 4966 } 4967 4968 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 4969 assert(VM_Version::supports_avx512bw(), ""); 4970 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 4971 InstructionMark im(this); 4972 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4973 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4974 attributes.reset_is_clear_context(); 4975 attributes.set_embedded_opmask_register_specifier(mask); 4976 attributes.set_is_evex_instruction(); 4977 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4978 emit_int8(0x74); 4979 emit_operand(as_Register(kdst->encoding()), src, 0); 4980 } 4981 4982 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4983 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 4984 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4985 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4986 emit_int16(0x75, (0xC0 | encode)); 4987 } 4988 4989 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4990 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4991 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4992 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4993 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4994 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4995 emit_int16(0x75, (0xC0 | encode)); 4996 } 4997 4998 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4999 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 5000 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 5001 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 5002 InstructionMark im(this); 5003 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5004 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5005 emit_int8(0x75); 5006 emit_operand(dst, src, 0); 5007 } 5008 5009 // In this context, kdst is written the mask used to process the equal components 5010 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 5011 assert(VM_Version::supports_avx512bw(), ""); 5012 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5013 attributes.set_is_evex_instruction(); 5014 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5015 emit_int16(0x75, (0xC0 | encode)); 5016 } 5017 5018 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 5019 assert(VM_Version::supports_avx512bw(), ""); 5020 InstructionMark im(this); 5021 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5022 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5023 attributes.set_is_evex_instruction(); 5024 int dst_enc = kdst->encoding(); 5025 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5026 emit_int8(0x75); 5027 emit_operand(as_Register(dst_enc), src, 0); 5028 } 5029 5030 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 5031 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) { 5032 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5033 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5034 emit_int16(0x76, (0xC0 | encode)); 5035 } 5036 5037 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 5038 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5039 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 5040 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 5041 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5043 emit_int16(0x76, (0xC0 | encode)); 5044 } 5045 5046 // In this context, kdst is written the mask used to process the equal components 5047 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) { 5048 assert(VM_Version::supports_evex(), ""); 5049 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5050 attributes.set_is_evex_instruction(); 5051 attributes.reset_is_clear_context(); 5052 attributes.set_embedded_opmask_register_specifier(mask); 5053 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5054 emit_int16(0x76, (0xC0 | encode)); 5055 } 5056 5057 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 5058 assert(VM_Version::supports_evex(), ""); 5059 InstructionMark im(this); 5060 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5061 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 5062 attributes.set_is_evex_instruction(); 5063 attributes.reset_is_clear_context(); 5064 attributes.set_embedded_opmask_register_specifier(mask); 5065 int dst_enc = kdst->encoding(); 5066 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5067 emit_int8(0x76); 5068 emit_operand(as_Register(dst_enc), src, 0); 5069 } 5070 5071 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 5072 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) { 5073 assert(VM_Version::supports_sse4_1(), ""); 5074 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5075 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5076 emit_int16(0x29, (0xC0 | encode)); 5077 } 5078 5079 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) { 5080 assert(VM_Version::supports_evex(), ""); 5081 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5082 attributes.set_is_evex_instruction(); 5083 attributes.reset_is_clear_context(); 5084 attributes.set_embedded_opmask_register_specifier(mask); 5085 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5086 emit_int16(0x29, (0xC0 | encode)); 5087 } 5088 5089 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) { 5090 assert(VM_Version::supports_avx(), ""); 5091 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5092 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5093 emit_int16(cond_encoding, (0xC0 | encode)); 5094 } 5095 5096 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 5097 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5098 assert(VM_Version::supports_avx(), ""); 5099 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5100 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5101 emit_int16(0x29, (0xC0 | encode)); 5102 } 5103 5104 // In this context, kdst is written the mask used to process the equal components 5105 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 5106 assert(VM_Version::supports_evex(), ""); 5107 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5108 attributes.reset_is_clear_context(); 5109 attributes.set_is_evex_instruction(); 5110 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5111 emit_int16(0x29, (0xC0 | encode)); 5112 } 5113 5114 // In this context, kdst is written the mask used to process the equal components 5115 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 5116 assert(VM_Version::supports_evex(), ""); 5117 InstructionMark im(this); 5118 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5119 attributes.reset_is_clear_context(); 5120 attributes.set_is_evex_instruction(); 5121 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 5122 int dst_enc = kdst->encoding(); 5123 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5124 emit_int8(0x29); 5125 emit_operand(as_Register(dst_enc), src, 0); 5126 } 5127 5128 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) { 5129 assert(VM_Version::supports_sse4_1(), ""); 5130 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5131 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5132 emit_int16(0x37, (0xC0 | encode)); 5133 } 5134 5135 void Assembler::pmovmskb(Register dst, XMMRegister src) { 5136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5137 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5138 emit_int16((unsigned char)0xD7, (0xC0 | encode)); 5139 } 5140 5141 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) { 5142 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) || 5143 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), ""); 5144 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5145 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5146 emit_int16((unsigned char)0xD7, (0xC0 | encode)); 5147 } 5148 5149 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) { 5150 assert(VM_Version::supports_avx(), ""); 5151 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5152 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 5153 emit_int16(0x50, (0xC0 | encode)); 5154 } 5155 5156 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) { 5157 assert(VM_Version::supports_avx(), ""); 5158 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5159 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5160 emit_int16(0x50, (0xC0 | encode)); 5161 } 5162 5163 5164 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) { 5165 assert(VM_Version::supports_sse4_1(), ""); 5166 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5167 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5168 emit_int24(0x16, (0xC0 | encode), imm8); 5169 } 5170 5171 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) { 5172 assert(VM_Version::supports_sse4_1(), ""); 5173 InstructionMark im(this); 5174 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5175 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 5176 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5177 emit_int8(0x16); 5178 emit_operand(src, dst, 1); 5179 emit_int8(imm8); 5180 } 5181 5182 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) { 5183 assert(VM_Version::supports_sse4_1(), ""); 5184 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5185 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5186 emit_int24(0x16, (0xC0 | encode), imm8); 5187 } 5188 5189 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) { 5190 assert(VM_Version::supports_sse4_1(), ""); 5191 InstructionMark im(this); 5192 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5193 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 5194 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5195 emit_int8(0x16); 5196 emit_operand(src, dst, 1); 5197 emit_int8(imm8); 5198 } 5199 5200 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) { 5201 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5202 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5203 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8); 5204 } 5205 5206 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) { 5207 assert(VM_Version::supports_sse4_1(), ""); 5208 InstructionMark im(this); 5209 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5210 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 5211 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5212 emit_int8(0x15); 5213 emit_operand(src, dst, 1); 5214 emit_int8(imm8); 5215 } 5216 5217 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) { 5218 assert(VM_Version::supports_sse4_1(), ""); 5219 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5220 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5221 emit_int24(0x14, (0xC0 | encode), imm8); 5222 } 5223 5224 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) { 5225 assert(VM_Version::supports_sse4_1(), ""); 5226 InstructionMark im(this); 5227 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5228 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 5229 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5230 emit_int8(0x14); 5231 emit_operand(src, dst, 1); 5232 emit_int8(imm8); 5233 } 5234 5235 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) { 5236 assert(VM_Version::supports_sse4_1(), ""); 5237 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5238 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5239 emit_int24(0x22, (0xC0 | encode), imm8); 5240 } 5241 5242 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) { 5243 assert(VM_Version::supports_sse4_1(), ""); 5244 InstructionMark im(this); 5245 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5246 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 5247 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5248 emit_int8(0x22); 5249 emit_operand(dst, src, 1); 5250 emit_int8(imm8); 5251 } 5252 5253 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 5254 assert(VM_Version::supports_avx(), ""); 5255 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5256 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5257 emit_int24(0x22, (0xC0 | encode), imm8); 5258 } 5259 5260 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) { 5261 assert(VM_Version::supports_sse4_1(), ""); 5262 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5263 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5264 emit_int24(0x22, (0xC0 | encode), imm8); 5265 } 5266 5267 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) { 5268 assert(VM_Version::supports_sse4_1(), ""); 5269 InstructionMark im(this); 5270 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5271 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 5272 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5273 emit_int8(0x22); 5274 emit_operand(dst, src, 1); 5275 emit_int8(imm8); 5276 } 5277 5278 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 5279 assert(VM_Version::supports_avx(), ""); 5280 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 5281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5282 emit_int24(0x22, (0xC0 | encode), imm8); 5283 } 5284 5285 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) { 5286 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5287 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 5288 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8); 5289 } 5290 5291 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) { 5292 InstructionMark im(this); 5293 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5294 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 5295 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5296 emit_int8((unsigned char)0xC4); 5297 emit_operand(dst, src, 1); 5298 emit_int8(imm8); 5299 } 5300 5301 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 5302 assert(VM_Version::supports_avx(), ""); 5303 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5304 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 5305 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8); 5306 } 5307 5308 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) { 5309 assert(VM_Version::supports_sse4_1(), ""); 5310 InstructionMark im(this); 5311 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5312 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 5313 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5314 emit_int8(0x20); 5315 emit_operand(dst, src, 1); 5316 emit_int8(imm8); 5317 } 5318 5319 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) { 5320 assert(VM_Version::supports_sse4_1(), ""); 5321 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5322 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5323 emit_int24(0x20, (0xC0 | encode), imm8); 5324 } 5325 5326 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 5327 assert(VM_Version::supports_avx(), ""); 5328 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 5329 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 5330 emit_int24(0x20, (0xC0 | encode), imm8); 5331 } 5332 5333 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) { 5334 assert(VM_Version::supports_sse4_1(), ""); 5335 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 5336 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5337 emit_int24(0x21, (0xC0 | encode), imm8); 5338 } 5339 5340 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 5341 assert(VM_Version::supports_avx(), ""); 5342 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 5343 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5344 emit_int24(0x21, (0xC0 | encode), imm8); 5345 } 5346 5347 void Assembler::pmovzxbw(XMMRegister dst, Address src) { 5348 assert(VM_Version::supports_sse4_1(), ""); 5349 InstructionMark im(this); 5350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5351 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 5352 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5353 emit_int8(0x30); 5354 emit_operand(dst, src, 0); 5355 } 5356 5357 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 5358 assert(VM_Version::supports_sse4_1(), ""); 5359 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5360 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5361 emit_int16(0x30, (0xC0 | encode)); 5362 } 5363 5364 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) { 5365 assert(VM_Version::supports_sse4_1(), ""); 5366 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5367 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5368 emit_int16(0x20, (0xC0 | encode)); 5369 } 5370 5371 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) { 5372 assert(VM_Version::supports_sse4_1(), ""); 5373 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5374 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5375 emit_int16(0x35, (0xC0 | encode)); 5376 } 5377 5378 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) { 5379 assert(VM_Version::supports_sse4_1(), ""); 5380 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5381 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5382 emit_int16(0x21, (0xC0 | encode)); 5383 } 5384 5385 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) { 5386 assert(VM_Version::supports_sse4_1(), ""); 5387 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5388 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5389 emit_int16(0x31, (0xC0 | encode)); 5390 } 5391 5392 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) { 5393 assert(VM_Version::supports_sse4_1(), ""); 5394 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5395 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5396 emit_int16(0x22, (0xC0 | encode)); 5397 } 5398 5399 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) { 5400 assert(VM_Version::supports_sse4_1(), ""); 5401 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5402 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5403 emit_int16(0x23, (0xC0 | encode)); 5404 } 5405 5406 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 5407 assert(VM_Version::supports_avx(), ""); 5408 InstructionMark im(this); 5409 assert(dst != xnoreg, "sanity"); 5410 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5411 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 5412 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5413 emit_int8(0x30); 5414 emit_operand(dst, src, 0); 5415 } 5416 5417 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { 5418 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5419 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5420 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5421 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5422 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5423 emit_int16(0x30, (unsigned char) (0xC0 | encode)); 5424 } 5425 5426 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) { 5427 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5428 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5429 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5430 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5431 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5432 emit_int16(0x20, (0xC0 | encode)); 5433 } 5434 5435 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) { 5436 assert(VM_Version::supports_avx512vlbw(), ""); 5437 assert(dst != xnoreg, "sanity"); 5438 InstructionMark im(this); 5439 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 5440 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 5441 attributes.set_embedded_opmask_register_specifier(mask); 5442 attributes.set_is_evex_instruction(); 5443 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5444 emit_int8(0x30); 5445 emit_operand(dst, src, 0); 5446 } 5447 5448 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 5449 assert(VM_Version::supports_avx512vl(), ""); 5450 assert(dst != xnoreg, "sanity"); 5451 InstructionMark im(this); 5452 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 5453 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit); 5454 attributes.set_embedded_opmask_register_specifier(mask); 5455 attributes.set_is_evex_instruction(); 5456 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5457 emit_int8(0x31); 5458 emit_operand(dst, src, 0); 5459 } 5460 5461 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) { 5462 evpmovzxbd(dst, k0, src, vector_len); 5463 } 5464 5465 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 5466 assert(VM_Version::supports_evex(), ""); 5467 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r 5468 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5469 attributes.set_is_evex_instruction(); 5470 attributes.set_embedded_opmask_register_specifier(mask); 5471 if (merge) { 5472 attributes.reset_is_clear_context(); 5473 } 5474 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5475 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 5476 } 5477 5478 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) { 5479 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 5480 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5481 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5482 emit_int16(0x35, (0xC0 | encode)); 5483 } 5484 5485 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) { 5486 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 5487 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5488 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5489 emit_int16(0x31, (0xC0 | encode)); 5490 } 5491 5492 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) { 5493 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 5494 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5495 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5496 emit_int16(0x32, (0xC0 | encode)); 5497 } 5498 5499 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) { 5500 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5501 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5502 VM_Version::supports_evex(), ""); 5503 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5504 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5505 emit_int16(0x21, (0xC0 | encode)); 5506 } 5507 5508 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) { 5509 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5510 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5511 VM_Version::supports_evex(), ""); 5512 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5513 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5514 emit_int16(0x22, (0xC0 | encode)); 5515 } 5516 5517 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) { 5518 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5519 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5520 VM_Version::supports_evex(), ""); 5521 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5522 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5523 emit_int16(0x23, (0xC0 | encode)); 5524 } 5525 5526 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) { 5527 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5528 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5529 VM_Version::supports_evex(), ""); 5530 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5531 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5532 emit_int16(0x24, (0xC0 | encode)); 5533 } 5534 5535 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) { 5536 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5537 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5538 VM_Version::supports_evex(), ""); 5539 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5540 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5541 emit_int16(0x25, (0xC0 | encode)); 5542 } 5543 5544 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) { 5545 assert(VM_Version::supports_avx512vlbw(), ""); 5546 assert(src != xnoreg, "sanity"); 5547 InstructionMark im(this); 5548 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5549 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 5550 attributes.set_is_evex_instruction(); 5551 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 5552 emit_int8(0x30); 5553 emit_operand(src, dst, 0); 5554 } 5555 5556 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) { 5557 assert(VM_Version::supports_avx512vlbw(), ""); 5558 assert(src != xnoreg, "sanity"); 5559 InstructionMark im(this); 5560 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5561 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 5562 attributes.reset_is_clear_context(); 5563 attributes.set_embedded_opmask_register_specifier(mask); 5564 attributes.set_is_evex_instruction(); 5565 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 5566 emit_int8(0x30); 5567 emit_operand(src, dst, 0); 5568 } 5569 5570 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) { 5571 assert(VM_Version::supports_evex(), ""); 5572 assert(src != xnoreg, "sanity"); 5573 InstructionMark im(this); 5574 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5575 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit); 5576 attributes.set_is_evex_instruction(); 5577 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 5578 emit_int8(0x31); 5579 emit_operand(src, dst, 0); 5580 } 5581 5582 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) { 5583 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5584 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5585 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " "); 5586 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5587 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5588 emit_int16(0x33, (0xC0 | encode)); 5589 } 5590 5591 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) { 5592 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5593 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5594 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " "); 5595 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5596 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5597 emit_int16(0x34, (0xC0 | encode)); 5598 } 5599 5600 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) { 5601 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5602 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5603 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 5604 } 5605 5606 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5607 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5608 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5609 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), ""); 5610 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5611 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5612 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 5613 } 5614 5615 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5616 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5617 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5618 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5619 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5620 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5621 emit_int16(0x04, (0xC0 | encode)); 5622 } 5623 5624 void Assembler::vpmadd52luq(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 5625 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), ""); 5626 5627 InstructionMark im(this); 5628 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5629 5630 if (VM_Version::supports_avx512ifma()) { 5631 attributes.set_is_evex_instruction(); 5632 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 5633 } 5634 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5635 emit_int8((unsigned char)0xB4); 5636 emit_operand(dst, src2, 0); 5637 } 5638 5639 void Assembler::vpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5640 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), ""); 5641 5642 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5643 5644 if (VM_Version::supports_avx512ifma()) { 5645 attributes.set_is_evex_instruction(); 5646 } 5647 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5648 emit_int16((unsigned char)0xB4, (0xC0 | encode)); 5649 } 5650 5651 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5652 evpmadd52luq(dst, k0, src1, src2, false, vector_len); 5653 } 5654 5655 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5656 assert(VM_Version::supports_avx512ifma(), ""); 5657 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5658 attributes.set_is_evex_instruction(); 5659 attributes.set_embedded_opmask_register_specifier(mask); 5660 if (merge) { 5661 attributes.reset_is_clear_context(); 5662 } 5663 5664 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5665 emit_int16((unsigned char)0xB4, (0xC0 | encode)); 5666 } 5667 5668 void Assembler::vpmadd52huq(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 5669 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), ""); 5670 5671 InstructionMark im(this); 5672 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5673 5674 if (VM_Version::supports_avx512ifma()) { 5675 attributes.set_is_evex_instruction(); 5676 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 5677 } 5678 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5679 emit_int8((unsigned char)0xB5); 5680 emit_operand(dst, src2, 0); 5681 } 5682 5683 void Assembler::vpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5684 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), ""); 5685 5686 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5687 5688 if (VM_Version::supports_avx512ifma()) { 5689 attributes.set_is_evex_instruction(); 5690 } 5691 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5692 emit_int16((unsigned char)0xB5, (0xC0 | encode)); 5693 } 5694 5695 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5696 evpmadd52huq(dst, k0, src1, src2, false, vector_len); 5697 } 5698 5699 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5700 assert(VM_Version::supports_avx512ifma(), ""); 5701 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5702 attributes.set_is_evex_instruction(); 5703 attributes.set_embedded_opmask_register_specifier(mask); 5704 if (merge) { 5705 attributes.reset_is_clear_context(); 5706 } 5707 5708 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5709 emit_int16((unsigned char)0xB5, (0xC0 | encode)); 5710 } 5711 5712 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5713 assert(VM_Version::supports_evex(), ""); 5714 assert(VM_Version::supports_avx512_vnni(), "must support vnni"); 5715 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5716 attributes.set_is_evex_instruction(); 5717 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5718 emit_int16(0x52, (0xC0 | encode)); 5719 } 5720 5721 // generic 5722 void Assembler::pop(Register dst) { 5723 int encode = prefix_and_encode(dst->encoding()); 5724 emit_int8(0x58 | encode); 5725 } 5726 5727 void Assembler::popcntl(Register dst, Address src) { 5728 assert(VM_Version::supports_popcnt(), "must support"); 5729 InstructionMark im(this); 5730 emit_int8((unsigned char)0xF3); 5731 prefix(src, dst, false, true /* is_map1 */); 5732 emit_int8((unsigned char)0xB8); 5733 emit_operand(dst, src, 0); 5734 } 5735 5736 void Assembler::epopcntl(Register dst, Address src, bool no_flags) { 5737 assert(VM_Version::supports_popcnt(), "must support"); 5738 InstructionMark im(this); 5739 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 5740 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 5741 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 5742 emit_int8((unsigned char)0x88); 5743 emit_operand(dst, src, 0); 5744 } 5745 5746 void Assembler::popcntl(Register dst, Register src) { 5747 assert(VM_Version::supports_popcnt(), "must support"); 5748 emit_int8((unsigned char)0xF3); 5749 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 5750 emit_opcode_prefix_and_encoding((unsigned char)0xB8, 0xC0, encode); 5751 } 5752 5753 void Assembler::epopcntl(Register dst, Register src, bool no_flags) { 5754 assert(VM_Version::supports_popcnt(), "must support"); 5755 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 5756 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 5757 emit_int16((unsigned char)0x88, (0xC0 | encode)); 5758 } 5759 5760 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5761 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature"); 5762 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5763 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5764 attributes.set_embedded_opmask_register_specifier(mask); 5765 attributes.set_is_evex_instruction(); 5766 if (merge) { 5767 attributes.reset_is_clear_context(); 5768 } 5769 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5770 emit_int16(0x54, (0xC0 | encode)); 5771 } 5772 5773 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5774 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature"); 5775 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5776 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5777 attributes.set_is_evex_instruction(); 5778 attributes.set_embedded_opmask_register_specifier(mask); 5779 if (merge) { 5780 attributes.reset_is_clear_context(); 5781 } 5782 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5783 emit_int16(0x54, (0xC0 | encode)); 5784 } 5785 5786 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5787 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature"); 5788 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5789 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5790 attributes.set_is_evex_instruction(); 5791 attributes.set_embedded_opmask_register_specifier(mask); 5792 if (merge) { 5793 attributes.reset_is_clear_context(); 5794 } 5795 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5796 emit_int16(0x55, (0xC0 | encode)); 5797 } 5798 5799 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5800 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature"); 5801 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5802 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5803 attributes.set_is_evex_instruction(); 5804 attributes.set_embedded_opmask_register_specifier(mask); 5805 if (merge) { 5806 attributes.reset_is_clear_context(); 5807 } 5808 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5809 emit_int16(0x55, (0xC0 | encode)); 5810 } 5811 5812 void Assembler::popf() { 5813 emit_int8((unsigned char)0x9D); 5814 } 5815 5816 void Assembler::prefetchnta(Address src) { 5817 InstructionMark im(this); 5818 prefix(src, true /* is_map1 */); 5819 emit_int8(0x18); 5820 emit_operand(rax, src, 0); // 0, src 5821 } 5822 5823 void Assembler::prefetchr(Address src) { 5824 assert(VM_Version::supports_3dnow_prefetch(), "must support"); 5825 InstructionMark im(this); 5826 prefix(src, true /* is_map1 */); 5827 emit_int8(0x0D); 5828 emit_operand(rax, src, 0); // 0, src 5829 } 5830 5831 void Assembler::prefetcht0(Address src) { 5832 InstructionMark im(this); 5833 prefix(src, true /* is_map1 */); 5834 emit_int8(0x18); 5835 emit_operand(rcx, src, 0); // 1, src 5836 } 5837 5838 void Assembler::prefetcht1(Address src) { 5839 InstructionMark im(this); 5840 prefix(src, true /* is_map1 */); 5841 emit_int8(0x18); 5842 emit_operand(rdx, src, 0); // 2, src 5843 } 5844 5845 void Assembler::prefetcht2(Address src) { 5846 InstructionMark im(this); 5847 prefix(src, true /* is_map1 */); 5848 emit_int8(0x18); 5849 emit_operand(rbx, src, 0); // 3, src 5850 } 5851 5852 void Assembler::prefetchw(Address src) { 5853 assert(VM_Version::supports_3dnow_prefetch(), "must support"); 5854 InstructionMark im(this); 5855 prefix(src, true /* is_map1 */); 5856 emit_int8(0x0D); 5857 emit_operand(rcx, src, 0); // 1, src 5858 } 5859 5860 void Assembler::prefix(Prefix p) { 5861 emit_int8(p); 5862 } 5863 5864 void Assembler::prefix16(int prefix) { 5865 assert(UseAPX, "APX features not enabled"); 5866 emit_int8((prefix & 0xff00) >> 8); 5867 emit_int8(prefix & 0xff); 5868 } 5869 5870 void Assembler::pshufb(XMMRegister dst, XMMRegister src) { 5871 assert(VM_Version::supports_ssse3(), ""); 5872 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5873 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5874 emit_int16(0x00, (0xC0 | encode)); 5875 } 5876 5877 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 5878 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 5879 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5880 attributes.set_is_evex_instruction(); 5881 attributes.set_embedded_opmask_register_specifier(mask); 5882 if (merge) { 5883 attributes.reset_is_clear_context(); 5884 } 5885 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5886 emit_int16(0x00, (0xC0 | encode)); 5887 } 5888 5889 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5890 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5891 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5892 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5893 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5894 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5895 emit_int16(0x00, (0xC0 | encode)); 5896 } 5897 5898 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 5899 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5900 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5901 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, ""); 5902 InstructionMark im(this); 5903 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5904 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5905 simd_prefix(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5906 emit_int8(0x00); 5907 emit_operand(dst, src, 0); 5908 } 5909 5910 void Assembler::pshufb(XMMRegister dst, Address src) { 5911 assert(VM_Version::supports_ssse3(), ""); 5912 InstructionMark im(this); 5913 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5914 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5915 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5916 emit_int8(0x00); 5917 emit_operand(dst, src, 0); 5918 } 5919 5920 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) { 5921 assert(isByte(mode), "invalid value"); 5922 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 5923 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5924 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5925 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5926 } 5927 5928 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5929 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5930 (vector_len == AVX_256bit? VM_Version::supports_avx2() : 5931 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), ""); 5932 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5933 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5934 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5935 } 5936 5937 void Assembler::pshufd(XMMRegister dst, Address src, int mode) { 5938 assert(isByte(mode), "invalid value"); 5939 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5940 InstructionMark im(this); 5941 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 5943 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5944 emit_int8(0x70); 5945 emit_operand(dst, src, 1); 5946 emit_int8(mode & 0xFF); 5947 } 5948 5949 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) { 5950 assert(isByte(mode), "invalid value"); 5951 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5952 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 5953 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5954 } 5955 5956 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5957 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5958 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5959 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), ""); 5960 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5961 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 5962 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5963 } 5964 5965 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 5966 assert(isByte(mode), "invalid value"); 5967 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5968 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5969 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5970 } 5971 5972 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) { 5973 assert(isByte(mode), "invalid value"); 5974 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5975 InstructionMark im(this); 5976 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5977 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5978 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5979 emit_int8(0x70); 5980 emit_operand(dst, src, 1); 5981 emit_int8(mode & 0xFF); 5982 } 5983 5984 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5985 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5986 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5987 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), ""); 5988 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5989 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5990 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5991 } 5992 5993 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 5994 assert(VM_Version::supports_evex(), "requires EVEX support"); 5995 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, ""); 5996 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5997 attributes.set_is_evex_instruction(); 5998 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5999 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF); 6000 } 6001 6002 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) { 6003 assert(isByte(imm8), "invalid value"); 6004 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6005 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6006 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 6007 } 6008 6009 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 6010 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6011 attributes.set_rex_vex_w_reverted(); 6012 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6013 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 6014 } 6015 6016 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) { 6017 assert(isByte(imm8), "invalid value"); 6018 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6019 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6020 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 6021 } 6022 6023 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 6024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6025 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6026 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 6027 } 6028 6029 void Assembler::psrldq(XMMRegister dst, int shift) { 6030 // Shift left 128 bit value in dst XMMRegister by shift number of bytes. 6031 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6032 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6033 emit_int24(0x73, (0xC0 | encode), shift); 6034 } 6035 6036 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 6037 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 6038 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 6039 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, ""); 6040 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6041 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6042 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 6043 } 6044 6045 void Assembler::pslldq(XMMRegister dst, int shift) { 6046 // Shift left 128 bit value in dst XMMRegister by shift number of bytes. 6047 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6048 // XMM7 is for /7 encoding: 66 0F 73 /7 ib 6049 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6050 emit_int24(0x73, (0xC0 | encode), shift); 6051 } 6052 6053 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 6054 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 6055 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 6056 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, ""); 6057 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6058 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6059 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 6060 } 6061 6062 void Assembler::ptest(XMMRegister dst, Address src) { 6063 assert(VM_Version::supports_sse4_1(), ""); 6064 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 6065 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs"); 6066 InstructionMark im(this); 6067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6068 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6069 emit_int8(0x17); 6070 emit_operand(dst, src, 0); 6071 } 6072 6073 void Assembler::ptest(XMMRegister dst, XMMRegister src) { 6074 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), ""); 6075 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6076 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6077 emit_int8(0x17); 6078 emit_int8((0xC0 | encode)); 6079 } 6080 6081 void Assembler::vptest(XMMRegister dst, Address src) { 6082 assert(VM_Version::supports_avx(), ""); 6083 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs"); 6084 InstructionMark im(this); 6085 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6086 assert(dst != xnoreg, "sanity"); 6087 // swap src<->dst for encoding 6088 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6089 emit_int8(0x17); 6090 emit_operand(dst, src, 0); 6091 } 6092 6093 void Assembler::vptest(XMMRegister dst, XMMRegister src) { 6094 assert(VM_Version::supports_avx(), ""); 6095 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6096 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6097 emit_int16(0x17, (0xC0 | encode)); 6098 } 6099 6100 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) { 6101 assert(VM_Version::supports_avx(), ""); 6102 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6103 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6104 emit_int16(0x17, (0xC0 | encode)); 6105 } 6106 6107 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) { 6108 assert(VM_Version::supports_avx(), ""); 6109 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6110 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6111 emit_int16(0x0E, (0xC0 | encode)); 6112 } 6113 6114 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6115 assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), ""); 6116 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r 6117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6118 attributes.set_is_evex_instruction(); 6119 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6120 emit_int16(0x26, (0xC0 | encode)); 6121 } 6122 6123 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6124 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), ""); 6125 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r 6126 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6127 attributes.set_is_evex_instruction(); 6128 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6129 emit_int16(0x27, (0xC0 | encode)); 6130 } 6131 6132 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6133 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), ""); 6134 // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r 6135 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6136 attributes.set_is_evex_instruction(); 6137 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 6138 emit_int16(0x27, (0xC0 | encode)); 6139 } 6140 6141 void Assembler::punpcklbw(XMMRegister dst, Address src) { 6142 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 6143 InstructionMark im(this); 6144 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true); 6145 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 6146 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6147 emit_int8(0x60); 6148 emit_operand(dst, src, 0); 6149 } 6150 6151 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) { 6152 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true); 6153 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6154 emit_int16(0x60, (0xC0 | encode)); 6155 } 6156 6157 void Assembler::punpckldq(XMMRegister dst, Address src) { 6158 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 6159 InstructionMark im(this); 6160 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6161 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 6162 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6163 emit_int8(0x62); 6164 emit_operand(dst, src, 0); 6165 } 6166 6167 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 6168 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6169 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6170 emit_int16(0x62, (0xC0 | encode)); 6171 } 6172 6173 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) { 6174 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6175 attributes.set_rex_vex_w_reverted(); 6176 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6177 emit_int16(0x6C, (0xC0 | encode)); 6178 } 6179 6180 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 6181 evpunpcklqdq(dst, k0, src1, src2, false, vector_len); 6182 } 6183 6184 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 6185 assert(VM_Version::supports_evex(), "requires AVX512F"); 6186 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 6187 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 6188 attributes.set_is_evex_instruction(); 6189 attributes.set_embedded_opmask_register_specifier(mask); 6190 if (merge) { 6191 attributes.reset_is_clear_context(); 6192 } 6193 6194 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6195 emit_int16(0x6C, (0xC0 | encode)); 6196 } 6197 6198 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 6199 evpunpckhqdq(dst, k0, src1, src2, false, vector_len); 6200 } 6201 6202 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 6203 assert(VM_Version::supports_evex(), "requires AVX512F"); 6204 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 6205 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 6206 attributes.set_is_evex_instruction(); 6207 attributes.set_embedded_opmask_register_specifier(mask); 6208 if (merge) { 6209 attributes.reset_is_clear_context(); 6210 } 6211 6212 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6213 emit_int16(0x6D, (0xC0 | encode)); 6214 } 6215 6216 void Assembler::push2(Register src1, Register src2, bool with_ppx) { 6217 assert(VM_Version::supports_apx_f(), "requires APX"); 6218 InstructionAttr attributes(0, /* rex_w */ with_ppx, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6219 /* EVEX.BASE */ 6220 int src_enc = src1->encoding(); 6221 /* EVEX.VVVV */ 6222 int nds_enc = src2->encoding(); 6223 6224 bool vex_b = (src_enc & 8) == 8; 6225 bool evex_v = (nds_enc >= 16); 6226 bool evex_b = (src_enc >= 16); 6227 6228 // EVEX.ND = 1; 6229 attributes.set_extended_context(); 6230 attributes.set_is_evex_instruction(); 6231 set_attributes(&attributes); 6232 6233 evex_prefix(0, vex_b, 0, 0, evex_b, evex_v, false /*eevex_x*/, nds_enc, VEX_SIMD_NONE, /* map4 */ VEX_OPCODE_0F_3C); 6234 emit_int16(0xFF, (0xC0 | (0x6 << 3) | (src_enc & 7))); 6235 } 6236 6237 void Assembler::pop2(Register src1, Register src2, bool with_ppx) { 6238 assert(VM_Version::supports_apx_f(), "requires APX"); 6239 InstructionAttr attributes(0, /* rex_w */ with_ppx, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6240 /* EVEX.BASE */ 6241 int src_enc = src1->encoding(); 6242 /* EVEX.VVVV */ 6243 int nds_enc = src2->encoding(); 6244 6245 bool vex_b = (src_enc & 8) == 8; 6246 bool evex_v = (nds_enc >= 16); 6247 bool evex_b = (src_enc >= 16); 6248 6249 // EVEX.ND = 1; 6250 attributes.set_extended_context(); 6251 attributes.set_is_evex_instruction(); 6252 set_attributes(&attributes); 6253 6254 evex_prefix(0, vex_b, 0, 0, evex_b, evex_v, false /*eevex_x*/, nds_enc, VEX_SIMD_NONE, /* map4 */ VEX_OPCODE_0F_3C); 6255 emit_int16(0x8F, (0xC0 | (src_enc & 7))); 6256 } 6257 6258 void Assembler::push2p(Register src1, Register src2) { 6259 push2(src1, src2, true); 6260 } 6261 6262 void Assembler::pop2p(Register src1, Register src2) { 6263 pop2(src1, src2, true); 6264 } 6265 6266 void Assembler::pushp(Register src) { 6267 assert(VM_Version::supports_apx_f(), "requires APX"); 6268 int encode = prefixq_and_encode_rex2(src->encoding()); 6269 emit_int8(0x50 | encode); 6270 } 6271 6272 void Assembler::popp(Register dst) { 6273 assert(VM_Version::supports_apx_f(), "requires APX"); 6274 int encode = prefixq_and_encode_rex2(dst->encoding()); 6275 emit_int8((unsigned char)0x58 | encode); 6276 } 6277 6278 void Assembler::push(int32_t imm32) { 6279 // in 64bits we push 64bits onto the stack but only 6280 // take a 32bit immediate 6281 emit_int8(0x68); 6282 emit_int32(imm32); 6283 } 6284 6285 void Assembler::push(Register src) { 6286 int encode = prefix_and_encode(src->encoding()); 6287 emit_int8(0x50 | encode); 6288 } 6289 6290 void Assembler::pushf() { 6291 emit_int8((unsigned char)0x9C); 6292 } 6293 6294 void Assembler::rcll(Register dst, int imm8) { 6295 assert(isShiftCount(imm8), "illegal shift count"); 6296 int encode = prefix_and_encode(dst->encoding()); 6297 if (imm8 == 1) { 6298 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 6299 } else { 6300 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 6301 } 6302 } 6303 6304 void Assembler::ercll(Register dst, Register src, int imm8) { 6305 assert(isShiftCount(imm8), "illegal shift count"); 6306 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6307 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes); 6308 if (imm8 == 1) { 6309 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 6310 } else { 6311 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 6312 } 6313 } 6314 6315 void Assembler::rcpps(XMMRegister dst, XMMRegister src) { 6316 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6317 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6318 emit_int16(0x53, (0xC0 | encode)); 6319 } 6320 6321 void Assembler::rcpss(XMMRegister dst, XMMRegister src) { 6322 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6323 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6324 emit_int16(0x53, (0xC0 | encode)); 6325 } 6326 6327 void Assembler::rdtsc() { 6328 emit_int16(0x0F, 0x31); 6329 } 6330 6331 // copies data from [esi] to [edi] using rcx pointer sized words 6332 // generic 6333 void Assembler::rep_mov() { 6334 // REP 6335 // MOVSQ 6336 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5); 6337 } 6338 6339 // sets rcx bytes with rax, value at [edi] 6340 void Assembler::rep_stosb() { 6341 // REP 6342 // STOSB 6343 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA); 6344 } 6345 6346 // sets rcx pointer sized words with rax, value at [edi] 6347 // generic 6348 void Assembler::rep_stos() { 6349 // REP 6350 // STOSQ 6351 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB); 6352 } 6353 6354 // scans rcx pointer sized words at [edi] for occurrence of rax, 6355 // generic 6356 void Assembler::repne_scan() { // repne_scan 6357 // SCASQ 6358 emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF); 6359 } 6360 6361 // scans rcx 4 byte words at [edi] for occurrence of rax, 6362 // generic 6363 void Assembler::repne_scanl() { // repne_scan 6364 // SCASL 6365 emit_int16((unsigned char)0xF2, (unsigned char)0xAF); 6366 } 6367 6368 void Assembler::ret(int imm16) { 6369 if (imm16 == 0) { 6370 emit_int8((unsigned char)0xC3); 6371 } else { 6372 emit_int8((unsigned char)0xC2); 6373 emit_int16(imm16); 6374 } 6375 } 6376 6377 void Assembler::roll(Register dst, int imm8) { 6378 assert(isShiftCount(imm8), "illegal shift count"); 6379 int encode = prefix_and_encode(dst->encoding()); 6380 if (imm8 == 1) { 6381 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 6382 } else { 6383 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 6384 } 6385 } 6386 6387 void Assembler::eroll(Register dst, Register src, int imm8, bool no_flags) { 6388 assert(isShiftCount(imm8), "illegal shift count"); 6389 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6390 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6391 if (imm8 == 1) { 6392 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 6393 } else { 6394 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 6395 } 6396 } 6397 6398 void Assembler::roll(Register dst) { 6399 int encode = prefix_and_encode(dst->encoding()); 6400 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 6401 } 6402 6403 void Assembler::eroll(Register dst, Register src, bool no_flags) { 6404 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6405 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6406 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 6407 } 6408 6409 void Assembler::rorl(Register dst, int imm8) { 6410 assert(isShiftCount(imm8), "illegal shift count"); 6411 int encode = prefix_and_encode(dst->encoding()); 6412 if (imm8 == 1) { 6413 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 6414 } else { 6415 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8); 6416 } 6417 } 6418 6419 void Assembler::erorl(Register dst, Register src, int imm8, bool no_flags) { 6420 assert(isShiftCount(imm8), "illegal shift count"); 6421 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6422 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6423 if (imm8 == 1) { 6424 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 6425 } else { 6426 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8); 6427 } 6428 } 6429 6430 void Assembler::rorl(Register dst) { 6431 int encode = prefix_and_encode(dst->encoding()); 6432 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 6433 } 6434 6435 void Assembler::erorl(Register dst, Register src, bool no_flags) { 6436 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6437 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6438 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 6439 } 6440 6441 void Assembler::rorq(Register dst) { 6442 int encode = prefixq_and_encode(dst->encoding()); 6443 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 6444 } 6445 6446 void Assembler::erorq(Register dst, Register src, bool no_flags) { 6447 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6448 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 6449 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 6450 } 6451 6452 void Assembler::rorq(Register dst, int imm8) { 6453 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 6454 int encode = prefixq_and_encode(dst->encoding()); 6455 if (imm8 == 1) { 6456 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 6457 } else { 6458 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8); 6459 } 6460 } 6461 6462 void Assembler::erorq(Register dst, Register src, int imm8, bool no_flags) { 6463 assert(isShiftCount(imm8), "illegal shift count"); 6464 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6465 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 6466 if (imm8 == 1) { 6467 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 6468 } else { 6469 emit_int24((unsigned char)0xC1, (0xC8 | encode), imm8); 6470 } 6471 } 6472 6473 void Assembler::rolq(Register dst) { 6474 int encode = prefixq_and_encode(dst->encoding()); 6475 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 6476 } 6477 6478 void Assembler::erolq(Register dst, Register src, bool no_flags) { 6479 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6480 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 6481 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 6482 } 6483 6484 void Assembler::rolq(Register dst, int imm8) { 6485 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 6486 int encode = prefixq_and_encode(dst->encoding()); 6487 if (imm8 == 1) { 6488 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 6489 } else { 6490 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 6491 } 6492 } 6493 6494 void Assembler::erolq(Register dst, Register src, int imm8, bool no_flags) { 6495 assert(isShiftCount(imm8), "illegal shift count"); 6496 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6497 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 6498 if (imm8 == 1) { 6499 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 6500 } else { 6501 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 6502 } 6503 } 6504 6505 void Assembler::sall(Address dst, int imm8) { 6506 InstructionMark im(this); 6507 assert(isShiftCount(imm8), "illegal shift count"); 6508 prefix(dst); 6509 if (imm8 == 1) { 6510 emit_int8((unsigned char)0xD1); 6511 emit_operand(as_Register(4), dst, 0); 6512 } 6513 else { 6514 emit_int8((unsigned char)0xC1); 6515 emit_operand(as_Register(4), dst, 1); 6516 emit_int8(imm8); 6517 } 6518 } 6519 6520 void Assembler::esall(Register dst, Address src, int imm8, bool no_flags) { 6521 InstructionMark im(this); 6522 assert(isShiftCount(imm8), "illegal shift count"); 6523 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6524 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6525 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6526 if (imm8 == 1) { 6527 emit_int8((unsigned char)0xD1); 6528 emit_operand(as_Register(4), src, 0); 6529 } 6530 else { 6531 emit_int8((unsigned char)0xC1); 6532 emit_operand(as_Register(4), src, 1); 6533 emit_int8(imm8); 6534 } 6535 } 6536 6537 void Assembler::sall(Address dst) { 6538 InstructionMark im(this); 6539 prefix(dst); 6540 emit_int8((unsigned char)0xD3); 6541 emit_operand(as_Register(4), dst, 0); 6542 } 6543 6544 void Assembler::esall(Register dst, Address src, bool no_flags) { 6545 InstructionMark im(this); 6546 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6547 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6548 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6549 emit_int8((unsigned char)0xD3); 6550 emit_operand(as_Register(4), src, 0); 6551 } 6552 6553 void Assembler::sall(Register dst, int imm8) { 6554 assert(isShiftCount(imm8), "illegal shift count"); 6555 int encode = prefix_and_encode(dst->encoding()); 6556 if (imm8 == 1) { 6557 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 6558 } else { 6559 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 6560 } 6561 } 6562 6563 void Assembler::esall(Register dst, Register src, int imm8, bool no_flags) { 6564 assert(isShiftCount(imm8), "illegal shift count"); 6565 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6566 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6567 if (imm8 == 1) { 6568 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 6569 } else { 6570 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 6571 } 6572 } 6573 6574 void Assembler::sall(Register dst) { 6575 int encode = prefix_and_encode(dst->encoding()); 6576 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 6577 } 6578 6579 void Assembler::esall(Register dst, Register src, bool no_flags) { 6580 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6581 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6582 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 6583 } 6584 6585 void Assembler::sarl(Address dst, int imm8) { 6586 assert(isShiftCount(imm8), "illegal shift count"); 6587 InstructionMark im(this); 6588 prefix(dst); 6589 if (imm8 == 1) { 6590 emit_int8((unsigned char)0xD1); 6591 emit_operand(as_Register(7), dst, 0); 6592 } 6593 else { 6594 emit_int8((unsigned char)0xC1); 6595 emit_operand(as_Register(7), dst, 1); 6596 emit_int8(imm8); 6597 } 6598 } 6599 6600 void Assembler::esarl(Register dst, Address src, int imm8, bool no_flags) { 6601 assert(isShiftCount(imm8), "illegal shift count"); 6602 InstructionMark im(this); 6603 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6604 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6605 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6606 if (imm8 == 1) { 6607 emit_int8((unsigned char)0xD1); 6608 emit_operand(as_Register(7), src, 0); 6609 } 6610 else { 6611 emit_int8((unsigned char)0xC1); 6612 emit_operand(as_Register(7), src, 1); 6613 emit_int8(imm8); 6614 } 6615 } 6616 6617 void Assembler::sarl(Address dst) { 6618 InstructionMark im(this); 6619 prefix(dst); 6620 emit_int8((unsigned char)0xD3); 6621 emit_operand(as_Register(7), dst, 0); 6622 } 6623 6624 void Assembler::esarl(Register dst, Address src, bool no_flags) { 6625 InstructionMark im(this); 6626 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6627 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6628 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6629 emit_int8((unsigned char)0xD3); 6630 emit_operand(as_Register(7), src, 0); 6631 } 6632 6633 void Assembler::sarl(Register dst, int imm8) { 6634 int encode = prefix_and_encode(dst->encoding()); 6635 assert(isShiftCount(imm8), "illegal shift count"); 6636 if (imm8 == 1) { 6637 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 6638 } else { 6639 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 6640 } 6641 } 6642 6643 void Assembler::esarl(Register dst, Register src, int imm8, bool no_flags) { 6644 assert(isShiftCount(imm8), "illegal shift count"); 6645 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6646 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6647 if (imm8 == 1) { 6648 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 6649 } else { 6650 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 6651 } 6652 } 6653 6654 void Assembler::sarl(Register dst) { 6655 int encode = prefix_and_encode(dst->encoding()); 6656 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 6657 } 6658 6659 void Assembler::esarl(Register dst, Register src, bool no_flags) { 6660 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6661 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6662 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 6663 } 6664 6665 void Assembler::sbbl(Address dst, int32_t imm32) { 6666 InstructionMark im(this); 6667 prefix(dst); 6668 emit_arith_operand(0x81, rbx, dst, imm32); 6669 } 6670 6671 void Assembler::sbbl(Register dst, int32_t imm32) { 6672 prefix(dst); 6673 emit_arith(0x81, 0xD8, dst, imm32); 6674 } 6675 6676 void Assembler::sbbl(Register dst, Address src) { 6677 InstructionMark im(this); 6678 prefix(src, dst); 6679 emit_int8(0x1B); 6680 emit_operand(dst, src, 0); 6681 } 6682 6683 void Assembler::sbbl(Register dst, Register src) { 6684 (void) prefix_and_encode(dst->encoding(), src->encoding()); 6685 emit_arith(0x1B, 0xC0, dst, src); 6686 } 6687 6688 void Assembler::setb(Condition cc, Register dst) { 6689 assert(0 <= cc && cc < 16, "illegal cc"); 6690 int encode = prefix_and_encode(dst->encoding(), true, true /* is_map1 */); 6691 emit_opcode_prefix_and_encoding((unsigned char)0x90 | cc, 0xC0, encode); 6692 } 6693 6694 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) { 6695 assert(VM_Version::supports_ssse3(), ""); 6696 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6697 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6698 emit_int24(0x0F, (0xC0 | encode), imm8); 6699 } 6700 6701 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 6702 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 6703 vector_len == AVX_256bit? VM_Version::supports_avx2() : 6704 0, ""); 6705 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 6706 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6707 emit_int24(0x0F, (0xC0 | encode), imm8); 6708 } 6709 6710 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 6711 assert(VM_Version::supports_evex(), ""); 6712 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6713 attributes.set_is_evex_instruction(); 6714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6715 emit_int24(0x3, (0xC0 | encode), imm8); 6716 } 6717 6718 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) { 6719 assert(VM_Version::supports_sse4_1(), ""); 6720 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6721 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6722 emit_int24(0x0E, (0xC0 | encode), imm8); 6723 } 6724 6725 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) { 6726 assert(VM_Version::supports_sha(), ""); 6727 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false); 6728 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8); 6729 } 6730 6731 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) { 6732 assert(VM_Version::supports_sha(), ""); 6733 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6734 emit_int16((unsigned char)0xC8, (0xC0 | encode)); 6735 } 6736 6737 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) { 6738 assert(VM_Version::supports_sha(), ""); 6739 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6740 emit_int16((unsigned char)0xC9, (0xC0 | encode)); 6741 } 6742 6743 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) { 6744 assert(VM_Version::supports_sha(), ""); 6745 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6746 emit_int16((unsigned char)0xCA, (0xC0 | encode)); 6747 } 6748 6749 // xmm0 is implicit additional source to this instruction. 6750 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) { 6751 assert(VM_Version::supports_sha(), ""); 6752 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6753 emit_int16((unsigned char)0xCB, (0xC0 | encode)); 6754 } 6755 6756 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) { 6757 assert(VM_Version::supports_sha(), ""); 6758 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6759 emit_int16((unsigned char)0xCC, (0xC0 | encode)); 6760 } 6761 6762 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) { 6763 assert(VM_Version::supports_sha(), ""); 6764 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 6765 emit_int16((unsigned char)0xCD, (0xC0 | encode)); 6766 } 6767 6768 void Assembler::sha512msg1(XMMRegister dst, XMMRegister src) { 6769 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), ""); 6770 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6771 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 6772 emit_int16((unsigned char)0xCC, (0xC0 | encode)); 6773 } 6774 6775 void Assembler::sha512msg2(XMMRegister dst, XMMRegister src) { 6776 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), ""); 6777 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6778 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 6779 emit_int16((unsigned char)0xCD, (0xC0 | encode)); 6780 } 6781 6782 void Assembler::sha512rnds2(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6783 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), ""); 6784 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6785 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 6786 emit_int16((unsigned char)0xCB, (0xC0 | encode)); 6787 } 6788 6789 void Assembler::shll(Register dst, int imm8) { 6790 assert(isShiftCount(imm8), "illegal shift count"); 6791 int encode = prefix_and_encode(dst->encoding()); 6792 if (imm8 == 1 ) { 6793 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 6794 } else { 6795 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 6796 } 6797 } 6798 6799 void Assembler::eshll(Register dst, Register src, int imm8, bool no_flags) { 6800 assert(isShiftCount(imm8), "illegal shift count"); 6801 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6802 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6803 if (imm8 == 1 ) { 6804 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 6805 } else { 6806 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 6807 } 6808 } 6809 6810 void Assembler::shll(Register dst) { 6811 int encode = prefix_and_encode(dst->encoding()); 6812 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 6813 } 6814 6815 void Assembler::eshll(Register dst, Register src, bool no_flags) { 6816 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6817 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6818 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 6819 } 6820 6821 void Assembler::shrl(Register dst, int imm8) { 6822 assert(isShiftCount(imm8), "illegal shift count"); 6823 int encode = prefix_and_encode(dst->encoding()); 6824 if (imm8 == 1) { 6825 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 6826 } 6827 else { 6828 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 6829 } 6830 } 6831 6832 void Assembler::eshrl(Register dst, Register src, int imm8, bool no_flags) { 6833 assert(isShiftCount(imm8), "illegal shift count"); 6834 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6835 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6836 if (imm8 == 1) { 6837 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 6838 } 6839 else { 6840 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 6841 } 6842 } 6843 6844 void Assembler::shrl(Register dst) { 6845 int encode = prefix_and_encode(dst->encoding()); 6846 emit_int16((unsigned char)0xD3, (0xE8 | encode)); 6847 } 6848 6849 void Assembler::eshrl(Register dst, Register src, bool no_flags) { 6850 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6851 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6852 emit_int16((unsigned char)0xD3, (0xE8 | encode)); 6853 } 6854 6855 void Assembler::shrl(Address dst) { 6856 InstructionMark im(this); 6857 prefix(dst); 6858 emit_int8((unsigned char)0xD3); 6859 emit_operand(as_Register(5), dst, 0); 6860 } 6861 6862 void Assembler::eshrl(Register dst, Address src, bool no_flags) { 6863 InstructionMark im(this); 6864 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6865 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6866 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6867 emit_int8((unsigned char)0xD3); 6868 emit_operand(as_Register(5), src, 0); 6869 } 6870 6871 void Assembler::shrl(Address dst, int imm8) { 6872 InstructionMark im(this); 6873 assert(isShiftCount(imm8), "illegal shift count"); 6874 prefix(dst); 6875 if (imm8 == 1) { 6876 emit_int8((unsigned char)0xD1); 6877 emit_operand(as_Register(5), dst, 0); 6878 } 6879 else { 6880 emit_int8((unsigned char)0xC1); 6881 emit_operand(as_Register(5), dst, 1); 6882 emit_int8(imm8); 6883 } 6884 } 6885 6886 void Assembler::eshrl(Register dst, Address src, int imm8, bool no_flags) { 6887 InstructionMark im(this); 6888 assert(isShiftCount(imm8), "illegal shift count"); 6889 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6890 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 6891 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 6892 if (imm8 == 1) { 6893 emit_int8((unsigned char)0xD1); 6894 emit_operand(as_Register(5), src, 0); 6895 } 6896 else { 6897 emit_int8((unsigned char)0xC1); 6898 emit_operand(as_Register(5), src, 1); 6899 emit_int8(imm8); 6900 } 6901 } 6902 6903 void Assembler::shldl(Register dst, Register src) { 6904 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6905 emit_opcode_prefix_and_encoding((unsigned char)0xA5, 0xC0, encode); 6906 } 6907 6908 void Assembler::eshldl(Register dst, Register src1, Register src2, bool no_flags) { 6909 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xA5, no_flags, true /* is_map1 */); 6910 } 6911 6912 void Assembler::shldl(Register dst, Register src, int8_t imm8) { 6913 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6914 emit_opcode_prefix_and_encoding((unsigned char)0xA4, 0xC0, encode, imm8); 6915 } 6916 6917 void Assembler::eshldl(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) { 6918 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x24, no_flags, true /* is_map1 */); 6919 } 6920 6921 void Assembler::shrdl(Register dst, Register src) { 6922 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6923 emit_opcode_prefix_and_encoding((unsigned char)0xAD, 0xC0, encode); 6924 } 6925 6926 void Assembler::eshrdl(Register dst, Register src1, Register src2, bool no_flags) { 6927 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAD, no_flags, true /* is_map1 */); 6928 } 6929 6930 void Assembler::shrdl(Register dst, Register src, int8_t imm8) { 6931 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6932 emit_opcode_prefix_and_encoding((unsigned char)0xAC, 0xC0, encode, imm8); 6933 } 6934 6935 void Assembler::eshrdl(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) { 6936 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x2C, no_flags, true /* is_map1 */); 6937 } 6938 6939 void Assembler::shldq(Register dst, Register src, int8_t imm8) { 6940 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6941 emit_opcode_prefix_and_encoding((unsigned char)0xA4, 0xC0, encode, imm8); 6942 } 6943 6944 void Assembler::eshldq(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) { 6945 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x24, no_flags, true /* is_map1 */); 6946 } 6947 6948 void Assembler::shrdq(Register dst, Register src, int8_t imm8) { 6949 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 6950 emit_opcode_prefix_and_encoding((unsigned char)0xAC, 0xC0, encode, imm8); 6951 } 6952 6953 void Assembler::eshrdq(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) { 6954 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x2C, no_flags, true /* is_map1 */); 6955 } 6956 6957 // copies a single word from [esi] to [edi] 6958 void Assembler::smovl() { 6959 emit_int8((unsigned char)0xA5); 6960 } 6961 6962 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) { 6963 assert(VM_Version::supports_sse4_1(), ""); 6964 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6965 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6966 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode); 6967 } 6968 6969 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) { 6970 assert(VM_Version::supports_sse4_1(), ""); 6971 InstructionMark im(this); 6972 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6973 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6974 emit_int8(0x0B); 6975 emit_operand(dst, src, 1); 6976 emit_int8((unsigned char)rmode); 6977 } 6978 6979 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 6980 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6981 attributes.set_rex_vex_w_reverted(); 6982 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6983 emit_int16(0x51, (0xC0 | encode)); 6984 } 6985 6986 void Assembler::sqrtsd(XMMRegister dst, Address src) { 6987 InstructionMark im(this); 6988 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6989 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6990 attributes.set_rex_vex_w_reverted(); 6991 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6992 emit_int8(0x51); 6993 emit_operand(dst, src, 0); 6994 } 6995 6996 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 6997 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6998 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6999 emit_int16(0x51, (0xC0 | encode)); 7000 } 7001 7002 void Assembler::std() { 7003 emit_int8((unsigned char)0xFD); 7004 } 7005 7006 void Assembler::sqrtss(XMMRegister dst, Address src) { 7007 InstructionMark im(this); 7008 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7010 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7011 emit_int8(0x51); 7012 emit_operand(dst, src, 0); 7013 } 7014 7015 void Assembler::stmxcsr(Address dst) { 7016 // This instruction should be SSE encoded with the REX2 prefix when an 7017 // extended GPR is present. To be consistent when UseAPX is enabled, use 7018 // this encoding even when an extended GPR is not used. 7019 if (UseAVX > 0 && !UseAPX ) { 7020 assert(VM_Version::supports_avx(), ""); 7021 InstructionMark im(this); 7022 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 7023 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7024 emit_int8((unsigned char)0xAE); 7025 emit_operand(as_Register(3), dst, 0); 7026 } else { 7027 InstructionMark im(this); 7028 prefix(dst, true /* is_map1 */); 7029 emit_int8((unsigned char)0xAE); 7030 emit_operand(as_Register(3), dst, 0); 7031 } 7032 } 7033 7034 void Assembler::subl(Address dst, int32_t imm32) { 7035 InstructionMark im(this); 7036 prefix(dst); 7037 emit_arith_operand(0x81, rbp, dst, imm32); 7038 } 7039 7040 void Assembler::esubl(Register dst, Address src, int32_t imm32, bool no_flags) { 7041 InstructionMark im(this); 7042 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7043 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 7044 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7045 emit_arith_operand(0x81, rbp, src, imm32); 7046 } 7047 7048 void Assembler::subl(Address dst, Register src) { 7049 InstructionMark im(this); 7050 prefix(dst, src); 7051 emit_int8(0x29); 7052 emit_operand(src, dst, 0); 7053 } 7054 7055 void Assembler::esubl(Register dst, Address src1, Register src2, bool no_flags) { 7056 InstructionMark im(this); 7057 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7058 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 7059 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7060 emit_int8(0x29); 7061 emit_operand(src2, src1, 0); 7062 } 7063 7064 void Assembler::subl(Register dst, int32_t imm32) { 7065 prefix(dst); 7066 emit_arith(0x81, 0xE8, dst, imm32); 7067 } 7068 7069 void Assembler::esubl(Register dst, Register src, int32_t imm32, bool no_flags) { 7070 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xE8, no_flags); 7071 } 7072 7073 // Force generation of a 4 byte immediate value even if it fits into 8bit 7074 void Assembler::subl_imm32(Register dst, int32_t imm32) { 7075 prefix(dst); 7076 emit_arith_imm32(0x81, 0xE8, dst, imm32); 7077 } 7078 7079 void Assembler::esubl_imm32(Register dst, Register src, int32_t imm32, bool no_flags) { 7080 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7081 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7082 emit_arith_imm32(0x81, 0xE8, src, imm32); 7083 } 7084 7085 void Assembler::subl(Register dst, Address src) { 7086 InstructionMark im(this); 7087 prefix(src, dst); 7088 emit_int8(0x2B); 7089 emit_operand(dst, src, 0); 7090 } 7091 7092 void Assembler::esubl(Register dst, Register src1, Address src2, bool no_flags) { 7093 InstructionMark im(this); 7094 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x2B, no_flags); 7095 } 7096 7097 void Assembler::subl(Register dst, Register src) { 7098 (void) prefix_and_encode(dst->encoding(), src->encoding()); 7099 emit_arith(0x2B, 0xC0, dst, src); 7100 } 7101 7102 void Assembler::esubl(Register dst, Register src1, Register src2, bool no_flags) { 7103 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7104 // NDD shares its encoding bits with NDS bits for regular EVEX instruction. 7105 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine. 7106 (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7107 emit_arith(0x2B, 0xC0, src1, src2); 7108 } 7109 7110 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 7111 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7112 attributes.set_rex_vex_w_reverted(); 7113 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7114 emit_int16(0x5C, (0xC0 | encode)); 7115 } 7116 7117 void Assembler::subsd(XMMRegister dst, Address src) { 7118 InstructionMark im(this); 7119 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7120 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7121 attributes.set_rex_vex_w_reverted(); 7122 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7123 emit_int8(0x5C); 7124 emit_operand(dst, src, 0); 7125 } 7126 7127 void Assembler::subss(XMMRegister dst, XMMRegister src) { 7128 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false); 7129 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7130 emit_int16(0x5C, (0xC0 | encode)); 7131 } 7132 7133 void Assembler::subss(XMMRegister dst, Address src) { 7134 InstructionMark im(this); 7135 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7136 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7137 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7138 emit_int8(0x5C); 7139 emit_operand(dst, src, 0); 7140 } 7141 7142 void Assembler::testb(Register dst, int imm8, bool use_ral) { 7143 if (dst == rax) { 7144 if (use_ral) { 7145 emit_int8((unsigned char)0xA8); 7146 emit_int8(imm8); 7147 } else { 7148 emit_int8((unsigned char)0xF6); 7149 emit_int8((unsigned char)0xC4); 7150 emit_int8(imm8); 7151 } 7152 } else { 7153 (void) prefix_and_encode(dst->encoding(), true); 7154 emit_arith_b(0xF6, 0xC0, dst, imm8); 7155 } 7156 } 7157 7158 void Assembler::testb(Address dst, int imm8) { 7159 InstructionMark im(this); 7160 prefix(dst); 7161 emit_int8((unsigned char)0xF6); 7162 emit_operand(rax, dst, 1); 7163 emit_int8(imm8); 7164 } 7165 7166 void Assembler::testl(Address dst, int32_t imm32) { 7167 InstructionMark im(this); 7168 prefix(dst); 7169 emit_int8((unsigned char)0xF7); 7170 emit_operand(as_Register(0), dst, 4); 7171 emit_int32(imm32); 7172 } 7173 7174 void Assembler::testl(Register dst, int32_t imm32) { 7175 // not using emit_arith because test 7176 // doesn't support sign-extension of 7177 // 8bit operands 7178 if (dst == rax) { 7179 emit_int8((unsigned char)0xA9); 7180 emit_int32(imm32); 7181 } else { 7182 int encode = dst->encoding(); 7183 encode = prefix_and_encode(encode); 7184 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 7185 emit_int32(imm32); 7186 } 7187 } 7188 7189 void Assembler::testl(Register dst, Register src) { 7190 (void) prefix_and_encode(dst->encoding(), src->encoding()); 7191 emit_arith(0x85, 0xC0, dst, src); 7192 } 7193 7194 void Assembler::testl(Register dst, Address src) { 7195 InstructionMark im(this); 7196 prefix(src, dst); 7197 emit_int8((unsigned char)0x85); 7198 emit_operand(dst, src, 0); 7199 } 7200 7201 void Assembler::tzcntl(Register dst, Register src) { 7202 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7203 emit_int8((unsigned char)0xF3); 7204 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 7205 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode); 7206 } 7207 7208 void Assembler::etzcntl(Register dst, Register src, bool no_flags) { 7209 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7210 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7211 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7212 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 7213 } 7214 7215 void Assembler::tzcntl(Register dst, Address src) { 7216 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7217 InstructionMark im(this); 7218 emit_int8((unsigned char)0xF3); 7219 prefix(src, dst, false, true /* is_map1 */); 7220 emit_int8((unsigned char)0xBC); 7221 emit_operand(dst, src, 0); 7222 } 7223 7224 void Assembler::etzcntl(Register dst, Address src, bool no_flags) { 7225 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7226 InstructionMark im(this); 7227 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7228 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 7229 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7230 emit_int8((unsigned char)0xF4); 7231 emit_operand(dst, src, 0); 7232 } 7233 7234 void Assembler::tzcntq(Register dst, Register src) { 7235 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7236 emit_int8((unsigned char)0xF3); 7237 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 7238 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode); 7239 } 7240 7241 void Assembler::etzcntq(Register dst, Register src, bool no_flags) { 7242 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7243 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7244 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7245 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 7246 } 7247 7248 void Assembler::tzcntq(Register dst, Address src) { 7249 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7250 InstructionMark im(this); 7251 emit_int8((unsigned char)0xF3); 7252 prefixq(src, dst, true /* is_map1 */); 7253 emit_int8((unsigned char)0xBC); 7254 emit_operand(dst, src, 0); 7255 } 7256 7257 void Assembler::etzcntq(Register dst, Address src, bool no_flags) { 7258 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 7259 InstructionMark im(this); 7260 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7261 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 7262 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7263 emit_int8((unsigned char)0xF4); 7264 emit_operand(dst, src, 0); 7265 } 7266 7267 void Assembler::ucomisd(XMMRegister dst, Address src) { 7268 InstructionMark im(this); 7269 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7270 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7271 attributes.set_rex_vex_w_reverted(); 7272 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7273 emit_int8(0x2E); 7274 emit_operand(dst, src, 0); 7275 } 7276 7277 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 7278 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7279 attributes.set_rex_vex_w_reverted(); 7280 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7281 emit_int16(0x2E, (0xC0 | encode)); 7282 } 7283 7284 void Assembler::ucomiss(XMMRegister dst, Address src) { 7285 InstructionMark im(this); 7286 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7287 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7288 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7289 emit_int8(0x2E); 7290 emit_operand(dst, src, 0); 7291 } 7292 7293 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 7294 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7295 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7296 emit_int16(0x2E, (0xC0 | encode)); 7297 } 7298 7299 void Assembler::xabort(int8_t imm8) { 7300 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF)); 7301 } 7302 7303 void Assembler::xaddb(Address dst, Register src) { 7304 InstructionMark im(this); 7305 prefix(dst, src, true, true /* is_map1 */); 7306 emit_int8((unsigned char)0xC0); 7307 emit_operand(src, dst, 0); 7308 } 7309 7310 void Assembler::xaddw(Address dst, Register src) { 7311 InstructionMark im(this); 7312 emit_int8(0x66); 7313 prefix(dst, src, false, true /* is_map1 */); 7314 emit_int8((unsigned char)0xC1); 7315 emit_operand(src, dst, 0); 7316 } 7317 7318 void Assembler::xaddl(Address dst, Register src) { 7319 InstructionMark im(this); 7320 prefix(dst, src, false, true /* is_map1 */); 7321 emit_int8((unsigned char)0xC1); 7322 emit_operand(src, dst, 0); 7323 } 7324 7325 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) { 7326 InstructionMark im(this); 7327 relocate(rtype); 7328 if (abort.is_bound()) { 7329 address entry = target(abort); 7330 assert(entry != nullptr, "abort entry null"); 7331 int offset = checked_cast<int>(entry - pc()); 7332 emit_int16((unsigned char)0xC7, (unsigned char)0xF8); 7333 emit_int32(offset - 6); // 2 opcode + 4 address 7334 } else { 7335 abort.add_patch_at(code(), locator()); 7336 emit_int16((unsigned char)0xC7, (unsigned char)0xF8); 7337 emit_int32(0); 7338 } 7339 } 7340 7341 void Assembler::xchgb(Register dst, Address src) { // xchg 7342 InstructionMark im(this); 7343 prefix(src, dst, true); 7344 emit_int8((unsigned char)0x86); 7345 emit_operand(dst, src, 0); 7346 } 7347 7348 void Assembler::xchgw(Register dst, Address src) { // xchg 7349 InstructionMark im(this); 7350 emit_int8(0x66); 7351 prefix(src, dst); 7352 emit_int8((unsigned char)0x87); 7353 emit_operand(dst, src, 0); 7354 } 7355 7356 void Assembler::xchgl(Register dst, Address src) { // xchg 7357 InstructionMark im(this); 7358 prefix(src, dst); 7359 emit_int8((unsigned char)0x87); 7360 emit_operand(dst, src, 0); 7361 } 7362 7363 void Assembler::xchgl(Register dst, Register src) { 7364 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 7365 emit_int16((unsigned char)0x87, (0xC0 | encode)); 7366 } 7367 7368 void Assembler::xend() { 7369 emit_int24(0x0F, 0x01, (unsigned char)0xD5); 7370 } 7371 7372 void Assembler::xgetbv() { 7373 emit_int24(0x0F, 0x01, (unsigned char)0xD0); 7374 } 7375 7376 void Assembler::xorl(Address dst, int32_t imm32) { 7377 InstructionMark im(this); 7378 prefix(dst); 7379 emit_arith_operand(0x81, as_Register(6), dst, imm32); 7380 } 7381 7382 void Assembler::exorl(Register dst, Address src, int32_t imm32, bool no_flags) { 7383 InstructionMark im(this); 7384 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7385 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 7386 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 7387 emit_arith_operand(0x81, as_Register(6), src, imm32); 7388 } 7389 7390 void Assembler::xorl(Register dst, int32_t imm32) { 7391 prefix(dst); 7392 emit_arith(0x81, 0xF0, dst, imm32); 7393 } 7394 7395 void Assembler::exorl(Register dst, Register src, int32_t imm32, bool no_flags) { 7396 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xF0, no_flags); 7397 } 7398 7399 void Assembler::xorl(Register dst, Address src) { 7400 InstructionMark im(this); 7401 prefix(src, dst); 7402 emit_int8(0x33); 7403 emit_operand(dst, src, 0); 7404 } 7405 7406 void Assembler::exorl(Register dst, Register src1, Address src2, bool no_flags) { 7407 InstructionMark im(this); 7408 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, no_flags); 7409 } 7410 7411 void Assembler::xorl(Register dst, Register src) { 7412 (void) prefix_and_encode(dst->encoding(), src->encoding()); 7413 emit_arith(0x33, 0xC0, dst, src); 7414 } 7415 7416 void Assembler::exorl(Register dst, Register src1, Register src2, bool no_flags) { 7417 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, 0xC0, no_flags, true /* is_commutative */); 7418 } 7419 7420 void Assembler::xorl(Address dst, Register src) { 7421 InstructionMark im(this); 7422 prefix(dst, src); 7423 emit_int8(0x31); 7424 emit_operand(src, dst, 0); 7425 } 7426 7427 void Assembler::exorl(Register dst, Address src1, Register src2, bool no_flags) { 7428 InstructionMark im(this); 7429 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */); 7430 } 7431 7432 void Assembler::xorb(Register dst, Address src) { 7433 InstructionMark im(this); 7434 prefix(src, dst); 7435 emit_int8(0x32); 7436 emit_operand(dst, src, 0); 7437 } 7438 7439 void Assembler::exorb(Register dst, Register src1, Address src2, bool no_flags) { 7440 InstructionMark im(this); 7441 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x32, no_flags); 7442 } 7443 7444 void Assembler::xorb(Address dst, Register src) { 7445 InstructionMark im(this); 7446 prefix(dst, src, true); 7447 emit_int8(0x30); 7448 emit_operand(src, dst, 0); 7449 } 7450 7451 void Assembler::exorb(Register dst, Address src1, Register src2, bool no_flags) { 7452 InstructionMark im(this); 7453 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x30, no_flags, false /* is_map1 */, true /* is_commutative */); 7454 } 7455 7456 void Assembler::xorw(Register dst, Address src) { 7457 InstructionMark im(this); 7458 emit_int8(0x66); 7459 prefix(src, dst); 7460 emit_int8(0x33); 7461 emit_operand(dst, src, 0); 7462 } 7463 7464 void Assembler::exorw(Register dst, Register src1, Address src2, bool no_flags) { 7465 InstructionMark im(this); 7466 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x33, no_flags); 7467 } 7468 7469 // AVX 3-operands scalar float-point arithmetic instructions 7470 7471 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) { 7472 assert(VM_Version::supports_avx(), ""); 7473 InstructionMark im(this); 7474 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7475 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7476 attributes.set_rex_vex_w_reverted(); 7477 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7478 emit_int8(0x58); 7479 emit_operand(dst, src, 0); 7480 } 7481 7482 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7483 assert(VM_Version::supports_avx(), ""); 7484 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7485 attributes.set_rex_vex_w_reverted(); 7486 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7487 emit_int16(0x58, (0xC0 | encode)); 7488 } 7489 7490 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) { 7491 assert(VM_Version::supports_avx(), ""); 7492 InstructionMark im(this); 7493 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7494 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7495 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7496 emit_int8(0x58); 7497 emit_operand(dst, src, 0); 7498 } 7499 7500 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7501 assert(VM_Version::supports_avx(), ""); 7502 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7503 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7504 emit_int16(0x58, (0xC0 | encode)); 7505 } 7506 7507 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) { 7508 assert(VM_Version::supports_avx(), ""); 7509 InstructionMark im(this); 7510 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7511 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7512 attributes.set_rex_vex_w_reverted(); 7513 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7514 emit_int8(0x5E); 7515 emit_operand(dst, src, 0); 7516 } 7517 7518 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7519 assert(VM_Version::supports_avx(), ""); 7520 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7521 attributes.set_rex_vex_w_reverted(); 7522 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7523 emit_int16(0x5E, (0xC0 | encode)); 7524 } 7525 7526 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) { 7527 assert(VM_Version::supports_avx(), ""); 7528 InstructionMark im(this); 7529 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7530 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7531 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7532 emit_int8(0x5E); 7533 emit_operand(dst, src, 0); 7534 } 7535 7536 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7537 assert(VM_Version::supports_avx(), ""); 7538 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7539 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7540 emit_int16(0x5E, (0xC0 | encode)); 7541 } 7542 7543 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 7544 assert(VM_Version::supports_fma(), ""); 7545 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7546 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7547 emit_int16((unsigned char)0xB9, (0xC0 | encode)); 7548 } 7549 7550 void Assembler::evfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2, EvexRoundPrefix rmode) { // Need to add rmode for rounding mode support 7551 assert(VM_Version::supports_evex(), ""); 7552 InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7553 attributes.set_extended_context(); 7554 attributes.set_is_evex_instruction(); 7555 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7556 emit_int16((unsigned char)0xAD, (0xC0 | encode)); 7557 } 7558 7559 void Assembler::vfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 7560 assert(VM_Version::supports_fma(), ""); 7561 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7562 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7563 emit_int16((unsigned char)0xAD, (0xC0 | encode)); 7564 } 7565 7566 void Assembler::vfnmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 7567 assert(VM_Version::supports_fma(), ""); 7568 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7569 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7570 emit_int16((unsigned char)0xBD, (0xC0 | encode)); 7571 } 7572 7573 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 7574 assert(VM_Version::supports_fma(), ""); 7575 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7576 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7577 emit_int16((unsigned char)0xB9, (0xC0 | encode)); 7578 } 7579 7580 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) { 7581 assert(VM_Version::supports_avx(), ""); 7582 InstructionMark im(this); 7583 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7584 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7585 attributes.set_rex_vex_w_reverted(); 7586 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7587 emit_int8(0x59); 7588 emit_operand(dst, src, 0); 7589 } 7590 7591 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7592 assert(VM_Version::supports_avx(), ""); 7593 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7594 attributes.set_rex_vex_w_reverted(); 7595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7596 emit_int16(0x59, (0xC0 | encode)); 7597 } 7598 7599 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) { 7600 assert(VM_Version::supports_avx(), ""); 7601 InstructionMark im(this); 7602 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7603 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7604 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7605 emit_int8(0x59); 7606 emit_operand(dst, src, 0); 7607 } 7608 7609 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7610 assert(VM_Version::supports_avx(), ""); 7611 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7612 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7613 emit_int16(0x59, (0xC0 | encode)); 7614 } 7615 7616 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) { 7617 assert(VM_Version::supports_avx(), ""); 7618 InstructionMark im(this); 7619 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7620 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 7621 attributes.set_rex_vex_w_reverted(); 7622 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7623 emit_int8(0x5C); 7624 emit_operand(dst, src, 0); 7625 } 7626 7627 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7628 assert(VM_Version::supports_avx(), ""); 7629 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7630 attributes.set_rex_vex_w_reverted(); 7631 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 7632 emit_int16(0x5C, (0xC0 | encode)); 7633 } 7634 7635 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) { 7636 assert(VM_Version::supports_avx(), ""); 7637 InstructionMark im(this); 7638 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7639 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 7640 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7641 emit_int8(0x5C); 7642 emit_operand(dst, src, 0); 7643 } 7644 7645 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 7646 assert(VM_Version::supports_avx(), ""); 7647 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7648 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 7649 emit_int16(0x5C, (0xC0 | encode)); 7650 } 7651 7652 //====================VECTOR ARITHMETIC===================================== 7653 7654 // Float-point vector arithmetic 7655 7656 void Assembler::addpd(XMMRegister dst, XMMRegister src) { 7657 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7658 attributes.set_rex_vex_w_reverted(); 7659 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7660 emit_int16(0x58, (0xC0 | encode)); 7661 } 7662 7663 void Assembler::addpd(XMMRegister dst, Address src) { 7664 InstructionMark im(this); 7665 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7666 attributes.set_rex_vex_w_reverted(); 7667 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7668 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7669 emit_int8(0x58); 7670 emit_operand(dst, src, 0); 7671 } 7672 7673 7674 void Assembler::addps(XMMRegister dst, XMMRegister src) { 7675 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7676 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7677 emit_int16(0x58, (0xC0 | encode)); 7678 } 7679 7680 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7681 assert(VM_Version::supports_avx(), ""); 7682 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7683 attributes.set_rex_vex_w_reverted(); 7684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7685 emit_int16(0x58, (0xC0 | encode)); 7686 } 7687 7688 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7689 assert(VM_Version::supports_avx(), ""); 7690 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7691 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7692 emit_int16(0x58, (0xC0 | encode)); 7693 } 7694 7695 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7696 assert(VM_Version::supports_avx(), ""); 7697 InstructionMark im(this); 7698 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7699 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7700 attributes.set_rex_vex_w_reverted(); 7701 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7702 emit_int8(0x58); 7703 emit_operand(dst, src, 0); 7704 } 7705 7706 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7707 assert(VM_Version::supports_avx(), ""); 7708 InstructionMark im(this); 7709 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7710 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7711 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7712 emit_int8(0x58); 7713 emit_operand(dst, src, 0); 7714 } 7715 7716 void Assembler::subpd(XMMRegister dst, XMMRegister src) { 7717 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7718 attributes.set_rex_vex_w_reverted(); 7719 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7720 emit_int16(0x5C, (0xC0 | encode)); 7721 } 7722 7723 void Assembler::subps(XMMRegister dst, XMMRegister src) { 7724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7725 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7726 emit_int16(0x5C, (0xC0 | encode)); 7727 } 7728 7729 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7730 assert(VM_Version::supports_avx(), ""); 7731 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7732 attributes.set_rex_vex_w_reverted(); 7733 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7734 emit_int16(0x5C, (0xC0 | encode)); 7735 } 7736 7737 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7738 assert(VM_Version::supports_avx(), ""); 7739 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7740 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7741 emit_int16(0x5C, (0xC0 | encode)); 7742 } 7743 7744 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7745 assert(VM_Version::supports_avx(), ""); 7746 InstructionMark im(this); 7747 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7748 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7749 attributes.set_rex_vex_w_reverted(); 7750 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7751 emit_int8(0x5C); 7752 emit_operand(dst, src, 0); 7753 } 7754 7755 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7756 assert(VM_Version::supports_avx(), ""); 7757 InstructionMark im(this); 7758 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7759 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7760 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7761 emit_int8(0x5C); 7762 emit_operand(dst, src, 0); 7763 } 7764 7765 void Assembler::mulpd(XMMRegister dst, XMMRegister src) { 7766 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7767 attributes.set_rex_vex_w_reverted(); 7768 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7769 emit_int16(0x59, (0xC0 | encode)); 7770 } 7771 7772 void Assembler::mulpd(XMMRegister dst, Address src) { 7773 InstructionMark im(this); 7774 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7775 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7776 attributes.set_rex_vex_w_reverted(); 7777 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7778 emit_int8(0x59); 7779 emit_operand(dst, src, 0); 7780 } 7781 7782 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 7783 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7784 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7785 emit_int16(0x59, (0xC0 | encode)); 7786 } 7787 7788 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7789 assert(VM_Version::supports_avx(), ""); 7790 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7791 attributes.set_rex_vex_w_reverted(); 7792 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7793 emit_int16(0x59, (0xC0 | encode)); 7794 } 7795 7796 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7797 assert(VM_Version::supports_avx(), ""); 7798 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7799 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7800 emit_int16(0x59, (0xC0 | encode)); 7801 } 7802 7803 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7804 assert(VM_Version::supports_avx(), ""); 7805 InstructionMark im(this); 7806 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7807 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7808 attributes.set_rex_vex_w_reverted(); 7809 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7810 emit_int8(0x59); 7811 emit_operand(dst, src, 0); 7812 } 7813 7814 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7815 assert(VM_Version::supports_avx(), ""); 7816 InstructionMark im(this); 7817 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7818 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7819 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7820 emit_int8(0x59); 7821 emit_operand(dst, src, 0); 7822 } 7823 7824 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 7825 assert(VM_Version::supports_fma(), ""); 7826 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7827 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7828 emit_int16((unsigned char)0xB8, (0xC0 | encode)); 7829 } 7830 7831 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 7832 assert(VM_Version::supports_fma(), ""); 7833 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7834 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7835 emit_int16((unsigned char)0xB8, (0xC0 | encode)); 7836 } 7837 7838 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 7839 assert(VM_Version::supports_fma(), ""); 7840 InstructionMark im(this); 7841 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7842 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7843 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7844 emit_int8((unsigned char)0xB8); 7845 emit_operand(dst, src2, 0); 7846 } 7847 7848 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 7849 assert(VM_Version::supports_fma(), ""); 7850 InstructionMark im(this); 7851 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7852 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7853 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7854 emit_int8((unsigned char)0xB8); 7855 emit_operand(dst, src2, 0); 7856 } 7857 7858 void Assembler::divpd(XMMRegister dst, XMMRegister src) { 7859 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7860 attributes.set_rex_vex_w_reverted(); 7861 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7862 emit_int16(0x5E, (0xC0 | encode)); 7863 } 7864 7865 void Assembler::divps(XMMRegister dst, XMMRegister src) { 7866 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7867 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7868 emit_int16(0x5E, (0xC0 | encode)); 7869 } 7870 7871 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7872 assert(VM_Version::supports_avx(), ""); 7873 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7874 attributes.set_rex_vex_w_reverted(); 7875 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7876 emit_int16(0x5E, (0xC0 | encode)); 7877 } 7878 7879 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7880 assert(VM_Version::supports_avx(), ""); 7881 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7882 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7883 emit_int16(0x5E, (0xC0 | encode)); 7884 } 7885 7886 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7887 assert(VM_Version::supports_avx(), ""); 7888 InstructionMark im(this); 7889 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7890 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7891 attributes.set_rex_vex_w_reverted(); 7892 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7893 emit_int8(0x5E); 7894 emit_operand(dst, src, 0); 7895 } 7896 7897 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7898 assert(VM_Version::supports_avx(), ""); 7899 InstructionMark im(this); 7900 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7901 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7902 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7903 emit_int8(0x5E); 7904 emit_operand(dst, src, 0); 7905 } 7906 7907 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) { 7908 assert(VM_Version::supports_avx(), ""); 7909 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 7910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7911 emit_int24(0x09, (0xC0 | encode), (rmode)); 7912 } 7913 7914 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) { 7915 assert(VM_Version::supports_avx(), ""); 7916 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs as BASE or INDEX of address operand"); 7917 InstructionMark im(this); 7918 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 7919 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7920 emit_int8(0x09); 7921 emit_operand(dst, src, 1); 7922 emit_int8((rmode)); 7923 } 7924 7925 void Assembler::vroundsd(XMMRegister dst, XMMRegister src, XMMRegister src2, int32_t rmode) { 7926 assert(VM_Version::supports_avx(), ""); 7927 assert(rmode <= 0x0f, "rmode 0x%x", rmode); 7928 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7929 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7930 emit_int24(0x0B, (0xC0 | encode), (rmode)); 7931 } 7932 7933 void Assembler::vrndscalesd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int32_t rmode) { 7934 assert(VM_Version::supports_evex(), "requires EVEX support"); 7935 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 7936 attributes.set_is_evex_instruction(); 7937 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7938 emit_int24(0x0B, (0xC0 | encode), (rmode)); 7939 } 7940 7941 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) { 7942 assert(VM_Version::supports_evex(), "requires EVEX support"); 7943 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7944 attributes.set_is_evex_instruction(); 7945 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7946 emit_int24(0x09, (0xC0 | encode), (rmode)); 7947 } 7948 7949 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) { 7950 assert(VM_Version::supports_evex(), "requires EVEX support"); 7951 assert(dst != xnoreg, "sanity"); 7952 InstructionMark im(this); 7953 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7954 attributes.set_is_evex_instruction(); 7955 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7956 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 7957 emit_int8(0x09); 7958 emit_operand(dst, src, 1); 7959 emit_int8((rmode)); 7960 } 7961 7962 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) { 7963 assert(VM_Version::supports_avx(), ""); 7964 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7965 attributes.set_rex_vex_w_reverted(); 7966 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7967 emit_int16(0x51, (0xC0 | encode)); 7968 } 7969 7970 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) { 7971 assert(VM_Version::supports_avx(), ""); 7972 InstructionMark im(this); 7973 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7974 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7975 attributes.set_rex_vex_w_reverted(); 7976 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7977 emit_int8(0x51); 7978 emit_operand(dst, src, 0); 7979 } 7980 7981 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) { 7982 assert(VM_Version::supports_avx(), ""); 7983 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7984 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7985 emit_int16(0x51, (0xC0 | encode)); 7986 } 7987 7988 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) { 7989 assert(VM_Version::supports_avx(), ""); 7990 InstructionMark im(this); 7991 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7992 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 7993 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7994 emit_int8(0x51); 7995 emit_operand(dst, src, 0); 7996 } 7997 7998 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 7999 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8000 attributes.set_rex_vex_w_reverted(); 8001 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8002 emit_int16(0x54, (0xC0 | encode)); 8003 } 8004 8005 void Assembler::andnpd(XMMRegister dst, XMMRegister src) { 8006 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8007 attributes.set_rex_vex_w_reverted(); 8008 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8009 emit_int16(0x55, (0xC0 | encode)); 8010 } 8011 8012 void Assembler::andps(XMMRegister dst, XMMRegister src) { 8013 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8014 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8015 emit_int16(0x54, (0xC0 | encode)); 8016 } 8017 8018 void Assembler::andps(XMMRegister dst, Address src) { 8019 InstructionMark im(this); 8020 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8021 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8022 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8023 emit_int8(0x54); 8024 emit_operand(dst, src, 0); 8025 } 8026 8027 void Assembler::andpd(XMMRegister dst, Address src) { 8028 InstructionMark im(this); 8029 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8030 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8031 attributes.set_rex_vex_w_reverted(); 8032 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8033 emit_int8(0x54); 8034 emit_operand(dst, src, 0); 8035 } 8036 8037 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8038 assert(VM_Version::supports_avx(), ""); 8039 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8040 attributes.set_rex_vex_w_reverted(); 8041 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8042 emit_int16(0x54, (0xC0 | encode)); 8043 } 8044 8045 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8046 assert(VM_Version::supports_avx(), ""); 8047 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8049 emit_int16(0x54, (0xC0 | encode)); 8050 } 8051 8052 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8053 assert(VM_Version::supports_avx(), ""); 8054 InstructionMark im(this); 8055 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8056 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8057 attributes.set_rex_vex_w_reverted(); 8058 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8059 emit_int8(0x54); 8060 emit_operand(dst, src, 0); 8061 } 8062 8063 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8064 assert(VM_Version::supports_avx(), ""); 8065 InstructionMark im(this); 8066 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8067 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8068 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8069 emit_int8(0x54); 8070 emit_operand(dst, src, 0); 8071 } 8072 8073 void Assembler::orpd(XMMRegister dst, XMMRegister src) { 8074 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 8075 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8076 attributes.set_rex_vex_w_reverted(); 8077 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8078 emit_int16(0x56, (0xC0 | encode)); 8079 } 8080 8081 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) { 8082 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8083 attributes.set_rex_vex_w_reverted(); 8084 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8085 emit_int8(0x15); 8086 emit_int8((0xC0 | encode)); 8087 } 8088 8089 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) { 8090 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8091 attributes.set_rex_vex_w_reverted(); 8092 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8093 emit_int16(0x14, (0xC0 | encode)); 8094 } 8095 8096 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 8097 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8098 attributes.set_rex_vex_w_reverted(); 8099 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8100 emit_int16(0x57, (0xC0 | encode)); 8101 } 8102 8103 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 8104 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8105 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8106 emit_int16(0x57, (0xC0 | encode)); 8107 } 8108 8109 void Assembler::xorpd(XMMRegister dst, Address src) { 8110 InstructionMark im(this); 8111 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8112 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8113 attributes.set_rex_vex_w_reverted(); 8114 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8115 emit_int8(0x57); 8116 emit_operand(dst, src, 0); 8117 } 8118 8119 void Assembler::xorps(XMMRegister dst, Address src) { 8120 InstructionMark im(this); 8121 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8122 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8123 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8124 emit_int8(0x57); 8125 emit_operand(dst, src, 0); 8126 } 8127 8128 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8129 assert(VM_Version::supports_avx(), ""); 8130 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8131 attributes.set_rex_vex_w_reverted(); 8132 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8133 emit_int16(0x57, (0xC0 | encode)); 8134 } 8135 8136 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8137 assert(VM_Version::supports_avx(), ""); 8138 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8139 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8140 emit_int16(0x57, (0xC0 | encode)); 8141 } 8142 8143 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8144 assert(VM_Version::supports_avx(), ""); 8145 InstructionMark im(this); 8146 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8147 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8148 attributes.set_rex_vex_w_reverted(); 8149 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8150 emit_int8(0x57); 8151 emit_operand(dst, src, 0); 8152 } 8153 8154 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8155 assert(VM_Version::supports_avx(), ""); 8156 InstructionMark im(this); 8157 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8158 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8159 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8160 emit_int8(0x57); 8161 emit_operand(dst, src, 0); 8162 } 8163 8164 // Integer vector arithmetic 8165 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8166 assert((VM_Version::supports_avx() && (vector_len == 0)) || 8167 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2"); 8168 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 8169 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8170 emit_int16(0x01, (0xC0 | encode)); 8171 } 8172 8173 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8174 assert((VM_Version::supports_avx() && (vector_len == 0)) || 8175 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2"); 8176 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 8177 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8178 emit_int16(0x02, (0xC0 | encode)); 8179 } 8180 8181 void Assembler::paddb(XMMRegister dst, XMMRegister src) { 8182 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8183 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8184 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 8185 } 8186 8187 void Assembler::paddw(XMMRegister dst, XMMRegister src) { 8188 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8189 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8190 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 8191 } 8192 8193 void Assembler::paddd(XMMRegister dst, XMMRegister src) { 8194 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8195 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8196 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 8197 } 8198 8199 void Assembler::paddd(XMMRegister dst, Address src) { 8200 InstructionMark im(this); 8201 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8202 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8203 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8204 emit_int8((unsigned char)0xFE); 8205 emit_operand(dst, src, 0); 8206 } 8207 8208 void Assembler::paddq(XMMRegister dst, XMMRegister src) { 8209 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8210 attributes.set_rex_vex_w_reverted(); 8211 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8212 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 8213 } 8214 8215 void Assembler::phaddw(XMMRegister dst, XMMRegister src) { 8216 assert(VM_Version::supports_sse3(), ""); 8217 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 8218 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8219 emit_int16(0x01, (0xC0 | encode)); 8220 } 8221 8222 void Assembler::phaddd(XMMRegister dst, XMMRegister src) { 8223 assert(VM_Version::supports_sse3(), ""); 8224 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 8225 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8226 emit_int16(0x02, (0xC0 | encode)); 8227 } 8228 8229 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8230 assert(UseAVX > 0, "requires some form of AVX"); 8231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8232 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8233 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 8234 } 8235 8236 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8237 assert(UseAVX > 0, "requires some form of AVX"); 8238 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8240 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 8241 } 8242 8243 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8244 assert(UseAVX > 0, "requires some form of AVX"); 8245 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8247 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 8248 } 8249 8250 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8251 assert(UseAVX > 0, "requires some form of AVX"); 8252 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8253 attributes.set_rex_vex_w_reverted(); 8254 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8255 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 8256 } 8257 8258 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8259 assert(UseAVX > 0, "requires some form of AVX"); 8260 InstructionMark im(this); 8261 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8262 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8263 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8264 emit_int8((unsigned char)0xFC); 8265 emit_operand(dst, src, 0); 8266 } 8267 8268 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8269 assert(UseAVX > 0, "requires some form of AVX"); 8270 InstructionMark im(this); 8271 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8272 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8273 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8274 emit_int8((unsigned char)0xFD); 8275 emit_operand(dst, src, 0); 8276 } 8277 8278 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8279 assert(UseAVX > 0, "requires some form of AVX"); 8280 InstructionMark im(this); 8281 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8282 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8283 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8284 emit_int8((unsigned char)0xFE); 8285 emit_operand(dst, src, 0); 8286 } 8287 8288 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8289 assert(UseAVX > 0, "requires some form of AVX"); 8290 InstructionMark im(this); 8291 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8292 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8293 attributes.set_rex_vex_w_reverted(); 8294 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8295 emit_int8((unsigned char)0xD4); 8296 emit_operand(dst, src, 0); 8297 } 8298 8299 void Assembler::vaddsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8300 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8301 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8302 attributes.set_is_evex_instruction(); 8303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8304 emit_int16(0x58, (0xC0 | encode)); 8305 } 8306 8307 void Assembler::vsubsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8308 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8309 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8310 attributes.set_is_evex_instruction(); 8311 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8312 emit_int16(0x5C, (0xC0 | encode)); 8313 } 8314 8315 void Assembler::vdivsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8316 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8317 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8318 attributes.set_is_evex_instruction(); 8319 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8320 emit_int16(0x5E, (0xC0 | encode)); 8321 } 8322 8323 void Assembler::vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8324 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8325 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8326 attributes.set_is_evex_instruction(); 8327 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8328 emit_int16(0x59, (0xC0 | encode)); 8329 } 8330 8331 void Assembler::vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8332 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8333 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8334 attributes.set_is_evex_instruction(); 8335 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8336 emit_int16(0x5F, (0xC0 | encode)); 8337 } 8338 8339 void Assembler::eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 8340 assert(VM_Version::supports_avx10_2(), ""); 8341 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8342 attributes.set_is_evex_instruction(); 8343 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes); 8344 emit_int24(0x53, (0xC0 | encode), imm8); 8345 } 8346 8347 void Assembler::vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src) { 8348 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8349 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8350 attributes.set_is_evex_instruction(); 8351 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8352 emit_int16(0x5D, (0xC0 | encode)); 8353 } 8354 8355 void Assembler::vsqrtsh(XMMRegister dst, XMMRegister src) { 8356 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 8357 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8358 attributes.set_is_evex_instruction(); 8359 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes); 8360 emit_int16(0x51, (0xC0 | encode)); 8361 } 8362 8363 void Assembler::vfmadd132sh(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 8364 assert(VM_Version::supports_avx512_fp16(), ""); 8365 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 8366 attributes.set_is_evex_instruction(); 8367 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes); 8368 emit_int16((unsigned char)0x99, (0xC0 | encode)); 8369 } 8370 8371 void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8372 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8373 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8374 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8375 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8376 emit_int16((unsigned char)0xEC, (0xC0 | encode)); 8377 } 8378 8379 void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8380 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8381 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8382 InstructionMark im(this); 8383 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8384 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8385 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8386 emit_int8((unsigned char)0xEC); 8387 emit_operand(dst, src, 0); 8388 } 8389 8390 void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8391 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8392 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8393 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8394 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8395 emit_int16((unsigned char)0xED, (0xC0 | encode)); 8396 } 8397 8398 void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8399 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8400 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8401 InstructionMark im(this); 8402 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8403 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8404 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8405 emit_int8((unsigned char)0xED); 8406 emit_operand(dst, src, 0); 8407 } 8408 8409 void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8410 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8411 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8412 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8413 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8414 emit_int16((unsigned char)0xDC, (0xC0 | encode)); 8415 } 8416 8417 void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8418 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8419 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8420 InstructionMark im(this); 8421 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8422 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8423 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8424 emit_int8((unsigned char)0xDC); 8425 emit_operand(dst, src, 0); 8426 } 8427 8428 8429 void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8430 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8431 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8432 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8433 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8434 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 8435 } 8436 8437 void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8438 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8439 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8440 InstructionMark im(this); 8441 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8442 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8443 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8444 emit_int8((unsigned char)0xDD); 8445 emit_operand(dst, src, 0); 8446 } 8447 8448 8449 void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8450 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8451 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8452 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8453 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8454 emit_int16((unsigned char)0xE8, (0xC0 | encode)); 8455 } 8456 8457 void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8458 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8459 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8460 InstructionMark im(this); 8461 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8462 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8463 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8464 emit_int8((unsigned char)0xE8); 8465 emit_operand(dst, src, 0); 8466 } 8467 8468 void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8469 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8470 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8471 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8472 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8473 emit_int16((unsigned char)0xE9, (0xC0 | encode)); 8474 } 8475 8476 void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8477 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8478 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8479 InstructionMark im(this); 8480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8481 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8482 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8483 emit_int8((unsigned char)0xE9); 8484 emit_operand(dst, src, 0); 8485 } 8486 8487 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8488 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8489 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8490 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8491 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8492 emit_int16((unsigned char)0xD8, (0xC0 | encode)); 8493 } 8494 8495 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8496 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8497 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8498 InstructionMark im(this); 8499 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8500 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8501 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8502 emit_int8((unsigned char)0xD8); 8503 emit_operand(dst, src, 0); 8504 } 8505 8506 void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8507 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8508 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8509 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8510 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8511 emit_int16((unsigned char)0xD9, (0xC0 | encode)); 8512 } 8513 8514 void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8515 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8516 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8517 InstructionMark im(this); 8518 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8519 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8520 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8521 emit_int8((unsigned char)0xD9); 8522 emit_operand(dst, src, 0); 8523 } 8524 8525 8526 void Assembler::psubb(XMMRegister dst, XMMRegister src) { 8527 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8528 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8529 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 8530 } 8531 8532 void Assembler::psubw(XMMRegister dst, XMMRegister src) { 8533 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8534 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8535 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 8536 } 8537 8538 void Assembler::psubd(XMMRegister dst, XMMRegister src) { 8539 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8540 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8541 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 8542 } 8543 8544 void Assembler::psubq(XMMRegister dst, XMMRegister src) { 8545 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8546 attributes.set_rex_vex_w_reverted(); 8547 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8548 emit_int8((unsigned char)0xFB); 8549 emit_int8((0xC0 | encode)); 8550 } 8551 8552 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8553 assert(UseAVX > 0, "requires some form of AVX"); 8554 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8555 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8556 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 8557 } 8558 8559 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8560 assert(UseAVX > 0, "requires some form of AVX"); 8561 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8562 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8563 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 8564 } 8565 8566 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8567 assert(UseAVX > 0, "requires some form of AVX"); 8568 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8569 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8570 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 8571 } 8572 8573 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8574 assert(UseAVX > 0, "requires some form of AVX"); 8575 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8576 attributes.set_rex_vex_w_reverted(); 8577 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8578 emit_int16((unsigned char)0xFB, (0xC0 | encode)); 8579 } 8580 8581 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8582 assert(UseAVX > 0, "requires some form of AVX"); 8583 InstructionMark im(this); 8584 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8585 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8586 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8587 emit_int8((unsigned char)0xF8); 8588 emit_operand(dst, src, 0); 8589 } 8590 8591 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8592 assert(UseAVX > 0, "requires some form of AVX"); 8593 InstructionMark im(this); 8594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8595 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8596 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8597 emit_int8((unsigned char)0xF9); 8598 emit_operand(dst, src, 0); 8599 } 8600 8601 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8602 assert(UseAVX > 0, "requires some form of AVX"); 8603 InstructionMark im(this); 8604 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8605 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8606 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8607 emit_int8((unsigned char)0xFA); 8608 emit_operand(dst, src, 0); 8609 } 8610 8611 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8612 assert(UseAVX > 0, "requires some form of AVX"); 8613 InstructionMark im(this); 8614 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8615 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8616 attributes.set_rex_vex_w_reverted(); 8617 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8618 emit_int8((unsigned char)0xFB); 8619 emit_operand(dst, src, 0); 8620 } 8621 8622 void Assembler::pmullw(XMMRegister dst, XMMRegister src) { 8623 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8624 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8625 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 8626 } 8627 8628 void Assembler::pmulld(XMMRegister dst, XMMRegister src) { 8629 assert(VM_Version::supports_sse4_1(), ""); 8630 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8631 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8632 emit_int16(0x40, (0xC0 | encode)); 8633 } 8634 8635 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) { 8636 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8637 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8638 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 8639 } 8640 8641 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8642 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) || 8643 (vector_len == AVX_256bit && VM_Version::supports_avx2()) || 8644 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), ""); 8645 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8646 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8647 emit_int16((unsigned char)0xE4, (0xC0 | encode)); 8648 } 8649 8650 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8651 assert(UseAVX > 0, "requires some form of AVX"); 8652 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8654 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 8655 } 8656 8657 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8658 assert(UseAVX > 0, "requires some form of AVX"); 8659 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8660 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8661 emit_int16(0x40, (0xC0 | encode)); 8662 } 8663 8664 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8665 assert(UseAVX > 2, "requires some form of EVEX"); 8666 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8667 attributes.set_is_evex_instruction(); 8668 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8669 emit_int16(0x40, (0xC0 | encode)); 8670 } 8671 8672 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8673 assert(UseAVX > 0, "requires some form of AVX"); 8674 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8675 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8676 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 8677 } 8678 8679 void Assembler::vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8680 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8681 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), ""); 8682 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8683 attributes.set_rex_vex_w_reverted(); 8684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8685 emit_int16(0x28, (0xC0 | encode)); 8686 } 8687 8688 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8689 assert(UseAVX > 0, "requires some form of AVX"); 8690 InstructionMark im(this); 8691 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8692 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8693 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8694 emit_int8((unsigned char)0xD5); 8695 emit_operand(dst, src, 0); 8696 } 8697 8698 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8699 assert(UseAVX > 0, "requires some form of AVX"); 8700 InstructionMark im(this); 8701 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8702 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8703 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8704 emit_int8(0x40); 8705 emit_operand(dst, src, 0); 8706 } 8707 8708 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8709 assert(UseAVX > 2, "requires some form of EVEX"); 8710 InstructionMark im(this); 8711 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 8712 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8713 attributes.set_is_evex_instruction(); 8714 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8715 emit_int8(0x40); 8716 emit_operand(dst, src, 0); 8717 } 8718 8719 // Min, max 8720 void Assembler::pminsb(XMMRegister dst, XMMRegister src) { 8721 assert(VM_Version::supports_sse4_1(), ""); 8722 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8723 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8724 emit_int16(0x38, (0xC0 | encode)); 8725 } 8726 8727 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8728 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8729 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 8730 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8731 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8732 emit_int16(0x38, (0xC0 | encode)); 8733 } 8734 8735 void Assembler::pminsw(XMMRegister dst, XMMRegister src) { 8736 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8737 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8738 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 8739 } 8740 8741 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8742 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8743 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 8744 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8745 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8746 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 8747 } 8748 8749 void Assembler::pminsd(XMMRegister dst, XMMRegister src) { 8750 assert(VM_Version::supports_sse4_1(), ""); 8751 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8752 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8753 emit_int16(0x39, (0xC0 | encode)); 8754 } 8755 8756 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8757 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8758 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), ""); 8759 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8760 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8761 emit_int16(0x39, (0xC0 | encode)); 8762 } 8763 8764 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8765 assert(UseAVX > 2, "requires AVX512F"); 8766 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8767 attributes.set_is_evex_instruction(); 8768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8769 emit_int16(0x39, (0xC0 | encode)); 8770 } 8771 8772 void Assembler::minps(XMMRegister dst, XMMRegister src) { 8773 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8774 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8775 emit_int16(0x5D, (0xC0 | encode)); 8776 } 8777 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8778 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 8779 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8780 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8781 emit_int16(0x5D, (0xC0 | encode)); 8782 } 8783 8784 void Assembler::minpd(XMMRegister dst, XMMRegister src) { 8785 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8786 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8787 emit_int16(0x5D, (0xC0 | encode)); 8788 } 8789 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8790 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 8791 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8792 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8793 emit_int16(0x5D, (0xC0 | encode)); 8794 } 8795 8796 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) { 8797 assert(VM_Version::supports_sse4_1(), ""); 8798 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8799 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8800 emit_int16(0x3C, (0xC0 | encode)); 8801 } 8802 8803 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8804 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8805 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 8806 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8807 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8808 emit_int16(0x3C, (0xC0 | encode)); 8809 } 8810 8811 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) { 8812 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8813 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8814 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 8815 } 8816 8817 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8818 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8819 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 8820 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8821 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8822 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 8823 } 8824 8825 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) { 8826 assert(VM_Version::supports_sse4_1(), ""); 8827 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8828 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8829 emit_int16(0x3D, (0xC0 | encode)); 8830 } 8831 8832 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8833 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8834 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), ""); 8835 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8837 emit_int16(0x3D, (0xC0 | encode)); 8838 } 8839 8840 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8841 assert(UseAVX > 2, "requires AVX512F"); 8842 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8843 attributes.set_is_evex_instruction(); 8844 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8845 emit_int16(0x3D, (0xC0 | encode)); 8846 } 8847 8848 void Assembler::maxps(XMMRegister dst, XMMRegister src) { 8849 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8850 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8851 emit_int16(0x5F, (0xC0 | encode)); 8852 } 8853 8854 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8855 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 8856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8857 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 8858 emit_int16(0x5F, (0xC0 | encode)); 8859 } 8860 8861 void Assembler::evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) { 8862 assert(VM_Version::supports_avx10_2(), ""); 8863 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8864 attributes.set_is_evex_instruction(); 8865 attributes.set_embedded_opmask_register_specifier(mask); 8866 if (merge) { 8867 attributes.reset_is_clear_context(); 8868 } 8869 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8870 emit_int24(0x52, (0xC0 | encode), imm8); 8871 } 8872 8873 void Assembler::evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) { 8874 assert(VM_Version::supports_avx10_2(), ""); 8875 InstructionMark im(this); 8876 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8877 attributes.set_is_evex_instruction(); 8878 attributes.set_embedded_opmask_register_specifier(mask); 8879 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8880 if (merge) { 8881 attributes.reset_is_clear_context(); 8882 } 8883 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8884 emit_int8(0x52); 8885 emit_operand(dst, src, 0); 8886 emit_int8(imm8); 8887 } 8888 8889 void Assembler::maxpd(XMMRegister dst, XMMRegister src) { 8890 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8891 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8892 emit_int16(0x5F, (0xC0 | encode)); 8893 } 8894 8895 void Assembler::evminmaxpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) { 8896 assert(VM_Version::supports_avx10_2(), ""); 8897 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8898 attributes.set_is_evex_instruction(); 8899 attributes.set_embedded_opmask_register_specifier(mask); 8900 if (merge) { 8901 attributes.reset_is_clear_context(); 8902 } 8903 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8904 emit_int24(0x52, (0xC0 | encode), imm8); 8905 } 8906 8907 void Assembler::evminmaxpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) { 8908 assert(VM_Version::supports_avx10_2(), ""); 8909 InstructionMark im(this); 8910 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8911 attributes.set_is_evex_instruction(); 8912 attributes.set_embedded_opmask_register_specifier(mask); 8913 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8914 if (merge) { 8915 attributes.reset_is_clear_context(); 8916 } 8917 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8918 emit_int8(0x52); 8919 emit_operand(dst, src, 0); 8920 emit_int8(imm8); 8921 } 8922 8923 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8924 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 8925 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8926 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8927 emit_int16(0x5F, (0xC0 | encode)); 8928 } 8929 8930 void Assembler::vpminub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8931 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8932 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), ""); 8933 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8935 emit_int16((unsigned char)0xDA, (0xC0 | encode)); 8936 } 8937 8938 void Assembler::vpminub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8939 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8940 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8941 InstructionMark im(this); 8942 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8943 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8944 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8945 emit_int8((unsigned char)0xDA); 8946 emit_operand(dst, src, 0); 8947 } 8948 8949 void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8950 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8951 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8952 attributes.set_is_evex_instruction(); 8953 attributes.set_embedded_opmask_register_specifier(mask); 8954 if (merge) { 8955 attributes.reset_is_clear_context(); 8956 } 8957 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8958 emit_int16((unsigned char)0xDA, (0xC0 | encode)); 8959 } 8960 8961 void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8962 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8963 InstructionMark im(this); 8964 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8965 attributes.set_is_evex_instruction(); 8966 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8967 attributes.set_embedded_opmask_register_specifier(mask); 8968 if (merge) { 8969 attributes.reset_is_clear_context(); 8970 } 8971 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8972 emit_int8((unsigned char)0xDA); 8973 emit_operand(dst, src, 0); 8974 } 8975 8976 void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8977 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 8978 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8979 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8980 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8981 emit_int16(0x3A, (0xC0 | encode)); 8982 } 8983 8984 void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8985 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 8986 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), ""); 8987 InstructionMark im(this); 8988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8989 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 8990 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8991 emit_int8((unsigned char)0x3A); 8992 emit_operand(dst, src, 0); 8993 } 8994 8995 void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8996 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8997 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8998 attributes.set_is_evex_instruction(); 8999 attributes.set_embedded_opmask_register_specifier(mask); 9000 if (merge) { 9001 attributes.reset_is_clear_context(); 9002 } 9003 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9004 emit_int16(0x3A, (0xC0 | encode)); 9005 } 9006 9007 void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9008 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9009 InstructionMark im(this); 9010 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9011 attributes.set_is_evex_instruction(); 9012 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 9013 attributes.set_embedded_opmask_register_specifier(mask); 9014 if (merge) { 9015 attributes.reset_is_clear_context(); 9016 } 9017 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9018 emit_int8(0x3A); 9019 emit_operand(dst, src, 0); 9020 } 9021 9022 void Assembler::vpminud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9023 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 9024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9025 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9026 emit_int16(0x3B, (0xC0 | encode)); 9027 } 9028 9029 void Assembler::vpminud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9030 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 9031 InstructionMark im(this); 9032 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9033 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9034 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9035 emit_int8((unsigned char)0x3B); 9036 emit_operand(dst, src, 0); 9037 } 9038 9039 void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9040 assert(VM_Version::supports_evex(), ""); 9041 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9042 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9043 attributes.set_is_evex_instruction(); 9044 attributes.set_embedded_opmask_register_specifier(mask); 9045 if (merge) { 9046 attributes.reset_is_clear_context(); 9047 } 9048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9049 emit_int16(0x3B, (0xC0 | encode)); 9050 } 9051 9052 void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9053 assert(VM_Version::supports_evex(), ""); 9054 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9055 InstructionMark im(this); 9056 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9057 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9058 attributes.set_is_evex_instruction(); 9059 attributes.set_embedded_opmask_register_specifier(mask); 9060 if (merge) { 9061 attributes.reset_is_clear_context(); 9062 } 9063 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9064 emit_int8(0x3B); 9065 emit_operand(dst, src, 0); 9066 } 9067 9068 void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9069 assert(VM_Version::supports_evex(), ""); 9070 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9071 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9072 attributes.set_is_evex_instruction(); 9073 attributes.set_embedded_opmask_register_specifier(mask); 9074 if (merge) { 9075 attributes.reset_is_clear_context(); 9076 } 9077 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9078 emit_int16(0x3B, (0xC0 | encode)); 9079 } 9080 9081 void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9082 assert(VM_Version::supports_evex(), ""); 9083 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9084 InstructionMark im(this); 9085 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9086 attributes.set_is_evex_instruction(); 9087 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9088 attributes.set_embedded_opmask_register_specifier(mask); 9089 if (merge) { 9090 attributes.reset_is_clear_context(); 9091 } 9092 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9093 emit_int8(0x3B); 9094 emit_operand(dst, src, 0); 9095 } 9096 9097 void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9098 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9099 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 9100 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 9101 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9102 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9103 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 9104 } 9105 9106 void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9107 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9108 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 9109 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 9110 InstructionMark im(this); 9111 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9112 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 9113 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9114 emit_int8((unsigned char)0xDE); 9115 emit_operand(dst, src, 0); 9116 } 9117 9118 void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9119 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9120 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9121 attributes.set_is_evex_instruction(); 9122 attributes.set_embedded_opmask_register_specifier(mask); 9123 if (merge) { 9124 attributes.reset_is_clear_context(); 9125 } 9126 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9127 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 9128 } 9129 9130 void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9131 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9132 InstructionMark im(this); 9133 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9134 attributes.set_is_evex_instruction(); 9135 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 9136 attributes.set_embedded_opmask_register_specifier(mask); 9137 if (merge) { 9138 attributes.reset_is_clear_context(); 9139 } 9140 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9141 emit_int8((unsigned char)0xDE); 9142 emit_operand(dst, src, 0); 9143 } 9144 9145 void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9146 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9147 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 9148 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 9149 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9150 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9151 emit_int16(0x3E, (0xC0 | encode)); 9152 } 9153 9154 void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9155 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9156 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 9157 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 9158 InstructionMark im(this); 9159 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9160 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 9161 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9162 emit_int8((unsigned char)0x3E); 9163 emit_operand(dst, src, 0); 9164 } 9165 9166 void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9167 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9168 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9169 attributes.set_is_evex_instruction(); 9170 attributes.set_embedded_opmask_register_specifier(mask); 9171 if (merge) { 9172 attributes.reset_is_clear_context(); 9173 } 9174 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9175 emit_int16(0x3E, (0xC0 | encode)); 9176 } 9177 9178 void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9179 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9180 InstructionMark im(this); 9181 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9182 attributes.set_is_evex_instruction(); 9183 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 9184 attributes.set_embedded_opmask_register_specifier(mask); 9185 if (merge) { 9186 attributes.reset_is_clear_context(); 9187 } 9188 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9189 emit_int8(0x3E); 9190 emit_operand(dst, src, 0); 9191 } 9192 9193 void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9194 assert(UseAVX > 0, ""); 9195 assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), ""); 9196 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9198 emit_int16(0x3F, (0xC0 | encode)); 9199 } 9200 9201 void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9202 assert(UseAVX > 0, ""); 9203 assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), ""); 9204 InstructionMark im(this); 9205 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9206 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9207 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9208 emit_int8((unsigned char)0x3F); 9209 emit_operand(dst, src, 0); 9210 } 9211 9212 void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9213 assert(VM_Version::supports_evex(), ""); 9214 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9216 attributes.set_is_evex_instruction(); 9217 attributes.set_embedded_opmask_register_specifier(mask); 9218 if (merge) { 9219 attributes.reset_is_clear_context(); 9220 } 9221 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9222 emit_int16(0x3F, (0xC0 | encode)); 9223 } 9224 9225 void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9226 assert(VM_Version::supports_evex(), ""); 9227 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9228 InstructionMark im(this); 9229 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9230 attributes.set_is_evex_instruction(); 9231 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9232 attributes.set_embedded_opmask_register_specifier(mask); 9233 if (merge) { 9234 attributes.reset_is_clear_context(); 9235 } 9236 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9237 emit_int8(0x3F); 9238 emit_operand(dst, src, 0); 9239 } 9240 9241 void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9242 assert(VM_Version::supports_evex(), ""); 9243 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9244 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9245 attributes.set_is_evex_instruction(); 9246 attributes.set_embedded_opmask_register_specifier(mask); 9247 if (merge) { 9248 attributes.reset_is_clear_context(); 9249 } 9250 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9251 emit_int16(0x3F, (0xC0 | encode)); 9252 } 9253 9254 void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9255 assert(VM_Version::supports_evex(), ""); 9256 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9257 InstructionMark im(this); 9258 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9259 attributes.set_is_evex_instruction(); 9260 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9261 attributes.set_embedded_opmask_register_specifier(mask); 9262 if (merge) { 9263 attributes.reset_is_clear_context(); 9264 } 9265 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9266 emit_int8(0x3F); 9267 emit_operand(dst, src, 0); 9268 } 9269 9270 // Shift packed integers left by specified number of bits. 9271 void Assembler::psllw(XMMRegister dst, int shift) { 9272 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9273 // XMM6 is for /6 encoding: 66 0F 71 /6 ib 9274 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9275 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9276 } 9277 9278 void Assembler::pslld(XMMRegister dst, int shift) { 9279 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9280 // XMM6 is for /6 encoding: 66 0F 72 /6 ib 9281 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9282 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9283 } 9284 9285 void Assembler::psllq(XMMRegister dst, int shift) { 9286 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9287 attributes.set_rex_vex_w_reverted(); 9288 // XMM6 is for /6 encoding: 66 0F 73 /6 ib 9289 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9290 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9291 } 9292 9293 void Assembler::psllw(XMMRegister dst, XMMRegister shift) { 9294 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9295 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9296 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 9297 } 9298 9299 void Assembler::pslld(XMMRegister dst, XMMRegister shift) { 9300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9301 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9302 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 9303 } 9304 9305 void Assembler::psllq(XMMRegister dst, XMMRegister shift) { 9306 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9307 attributes.set_rex_vex_w_reverted(); 9308 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9309 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 9310 } 9311 9312 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9313 assert(UseAVX > 0, "requires some form of AVX"); 9314 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9315 // XMM6 is for /6 encoding: 66 0F 71 /6 ib 9316 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9317 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9318 } 9319 9320 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9321 assert(UseAVX > 0, "requires some form of AVX"); 9322 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9323 // XMM6 is for /6 encoding: 66 0F 72 /6 ib 9324 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9325 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9326 } 9327 9328 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9329 assert(UseAVX > 0, "requires some form of AVX"); 9330 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9331 attributes.set_rex_vex_w_reverted(); 9332 // XMM6 is for /6 encoding: 66 0F 73 /6 ib 9333 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9334 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9335 } 9336 9337 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9338 assert(UseAVX > 0, "requires some form of AVX"); 9339 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9340 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9341 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 9342 } 9343 9344 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9345 assert(UseAVX > 0, "requires some form of AVX"); 9346 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9347 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9348 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 9349 } 9350 9351 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9352 assert(UseAVX > 0, "requires some form of AVX"); 9353 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9354 attributes.set_rex_vex_w_reverted(); 9355 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9356 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 9357 } 9358 9359 // Shift packed integers logically right by specified number of bits. 9360 void Assembler::psrlw(XMMRegister dst, int shift) { 9361 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9362 // XMM2 is for /2 encoding: 66 0F 71 /2 ib 9363 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9364 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9365 } 9366 9367 void Assembler::psrld(XMMRegister dst, int shift) { 9368 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9369 // XMM2 is for /2 encoding: 66 0F 72 /2 ib 9370 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9371 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9372 } 9373 9374 void Assembler::psrlq(XMMRegister dst, int shift) { 9375 // Do not confuse it with psrldq SSE2 instruction which 9376 // shifts 128 bit value in xmm register by number of bytes. 9377 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9378 attributes.set_rex_vex_w_reverted(); 9379 // XMM2 is for /2 encoding: 66 0F 73 /2 ib 9380 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9381 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9382 } 9383 9384 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) { 9385 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9386 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9387 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 9388 } 9389 9390 void Assembler::psrld(XMMRegister dst, XMMRegister shift) { 9391 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9392 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9393 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 9394 } 9395 9396 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) { 9397 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9398 attributes.set_rex_vex_w_reverted(); 9399 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9400 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 9401 } 9402 9403 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9404 assert(UseAVX > 0, "requires some form of AVX"); 9405 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9406 // XMM2 is for /2 encoding: 66 0F 71 /2 ib 9407 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9408 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9409 } 9410 9411 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9412 assert(UseAVX > 0, "requires some form of AVX"); 9413 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9414 // XMM2 is for /2 encoding: 66 0F 72 /2 ib 9415 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9416 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9417 } 9418 9419 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9420 assert(UseAVX > 0, "requires some form of AVX"); 9421 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9422 attributes.set_rex_vex_w_reverted(); 9423 // XMM2 is for /2 encoding: 66 0F 73 /2 ib 9424 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9425 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9426 } 9427 9428 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9429 assert(UseAVX > 0, "requires some form of AVX"); 9430 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9431 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9432 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 9433 } 9434 9435 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9436 assert(UseAVX > 0, "requires some form of AVX"); 9437 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9438 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9439 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 9440 } 9441 9442 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9443 assert(UseAVX > 0, "requires some form of AVX"); 9444 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9445 attributes.set_rex_vex_w_reverted(); 9446 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9447 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 9448 } 9449 9450 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9451 assert(VM_Version::supports_avx512bw(), ""); 9452 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9453 attributes.set_is_evex_instruction(); 9454 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9455 emit_int16(0x10, (0xC0 | encode)); 9456 } 9457 9458 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9459 assert(VM_Version::supports_avx512bw(), ""); 9460 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9461 attributes.set_is_evex_instruction(); 9462 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9463 emit_int16(0x12, (0xC0 | encode)); 9464 } 9465 9466 // Shift packed integers arithmetically right by specified number of bits. 9467 void Assembler::psraw(XMMRegister dst, int shift) { 9468 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9469 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 9470 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9471 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9472 } 9473 9474 void Assembler::psrad(XMMRegister dst, int shift) { 9475 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9476 // XMM4 is for /4 encoding: 66 0F 72 /4 ib 9477 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9478 emit_int8(0x72); 9479 emit_int8((0xC0 | encode)); 9480 emit_int8(shift & 0xFF); 9481 } 9482 9483 void Assembler::psraw(XMMRegister dst, XMMRegister shift) { 9484 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9485 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9486 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 9487 } 9488 9489 void Assembler::psrad(XMMRegister dst, XMMRegister shift) { 9490 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9491 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9492 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 9493 } 9494 9495 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9496 assert(UseAVX > 0, "requires some form of AVX"); 9497 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9498 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 9499 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9500 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9501 } 9502 9503 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9504 assert(UseAVX > 0, "requires some form of AVX"); 9505 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9506 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 9507 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9508 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9509 } 9510 9511 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9512 assert(UseAVX > 0, "requires some form of AVX"); 9513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 9514 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9515 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 9516 } 9517 9518 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9519 assert(UseAVX > 0, "requires some form of AVX"); 9520 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9521 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9522 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 9523 } 9524 9525 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9526 assert(UseAVX > 2, "requires AVX512"); 9527 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl"); 9528 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9529 attributes.set_is_evex_instruction(); 9530 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9531 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF); 9532 } 9533 9534 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9535 assert(UseAVX > 2, "requires AVX512"); 9536 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl"); 9537 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9538 attributes.set_is_evex_instruction(); 9539 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9540 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 9541 } 9542 9543 // logical operations packed integers 9544 void Assembler::pand(XMMRegister dst, XMMRegister src) { 9545 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9546 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9547 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 9548 } 9549 9550 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9551 assert(UseAVX > 0, "requires some form of AVX"); 9552 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9553 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9554 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 9555 } 9556 9557 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9558 assert(UseAVX > 0, "requires some form of AVX"); 9559 InstructionMark im(this); 9560 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9561 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9562 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9563 emit_int8((unsigned char)0xDB); 9564 emit_operand(dst, src, 0); 9565 } 9566 9567 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9568 evpandq(dst, k0, nds, src, false, vector_len); 9569 } 9570 9571 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9572 evpandq(dst, k0, nds, src, false, vector_len); 9573 } 9574 9575 //Variable Shift packed integers logically left. 9576 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9577 assert(UseAVX > 1, "requires AVX2"); 9578 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9579 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9580 emit_int16(0x47, (0xC0 | encode)); 9581 } 9582 9583 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9584 assert(UseAVX > 1, "requires AVX2"); 9585 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9586 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9587 emit_int16(0x47, (0xC0 | encode)); 9588 } 9589 9590 //Variable Shift packed integers logically right. 9591 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9592 assert(UseAVX > 1, "requires AVX2"); 9593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9594 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9595 emit_int16(0x45, (0xC0 | encode)); 9596 } 9597 9598 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9599 assert(UseAVX > 1, "requires AVX2"); 9600 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9601 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9602 emit_int16(0x45, (0xC0 | encode)); 9603 } 9604 9605 //Variable right Shift arithmetic packed integers . 9606 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9607 assert(UseAVX > 1, "requires AVX2"); 9608 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9609 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9610 emit_int16(0x46, (0xC0 | encode)); 9611 } 9612 9613 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9614 assert(VM_Version::supports_avx512bw(), ""); 9615 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9616 attributes.set_is_evex_instruction(); 9617 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9618 emit_int16(0x11, (0xC0 | encode)); 9619 } 9620 9621 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9622 assert(UseAVX > 2, "requires AVX512"); 9623 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 9624 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9625 attributes.set_is_evex_instruction(); 9626 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9627 emit_int16(0x46, (0xC0 | encode)); 9628 } 9629 9630 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9631 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2"); 9632 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9633 attributes.set_is_evex_instruction(); 9634 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9635 emit_int16(0x71, (0xC0 | encode)); 9636 } 9637 9638 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9639 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2"); 9640 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9641 attributes.set_is_evex_instruction(); 9642 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9643 emit_int16(0x73, (0xC0 | encode)); 9644 } 9645 9646 void Assembler::pandn(XMMRegister dst, XMMRegister src) { 9647 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9648 attributes.set_rex_vex_w_reverted(); 9649 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9650 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 9651 } 9652 9653 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9654 assert(UseAVX > 0, "requires some form of AVX"); 9655 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9656 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9657 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 9658 } 9659 9660 void Assembler::por(XMMRegister dst, XMMRegister src) { 9661 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9663 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 9664 } 9665 9666 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9667 assert(UseAVX > 0, "requires some form of AVX"); 9668 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9669 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9670 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 9671 } 9672 9673 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9674 assert(UseAVX > 0, "requires some form of AVX"); 9675 InstructionMark im(this); 9676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9677 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9678 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9679 emit_int8((unsigned char)0xEB); 9680 emit_operand(dst, src, 0); 9681 } 9682 9683 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9684 evporq(dst, k0, nds, src, false, vector_len); 9685 } 9686 9687 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9688 evporq(dst, k0, nds, src, false, vector_len); 9689 } 9690 9691 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9692 assert(VM_Version::supports_evex(), ""); 9693 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r 9694 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9695 attributes.set_is_evex_instruction(); 9696 attributes.set_embedded_opmask_register_specifier(mask); 9697 if (merge) { 9698 attributes.reset_is_clear_context(); 9699 } 9700 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9701 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 9702 } 9703 9704 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9705 assert(VM_Version::supports_evex(), ""); 9706 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r 9707 InstructionMark im(this); 9708 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9709 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9710 attributes.set_is_evex_instruction(); 9711 attributes.set_embedded_opmask_register_specifier(mask); 9712 if (merge) { 9713 attributes.reset_is_clear_context(); 9714 } 9715 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9716 emit_int8((unsigned char)0xEB); 9717 emit_operand(dst, src, 0); 9718 } 9719 9720 void Assembler::pxor(XMMRegister dst, XMMRegister src) { 9721 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9722 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9723 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9724 } 9725 9726 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9727 assert(UseAVX > 0, "requires some form of AVX"); 9728 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9729 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 9730 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, ""); 9731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9732 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9733 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9734 } 9735 9736 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9737 assert(UseAVX > 0, "requires some form of AVX"); 9738 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 9739 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 9740 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, ""); 9741 InstructionMark im(this); 9742 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9744 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9745 emit_int8((unsigned char)0xEF); 9746 emit_operand(dst, src, 0); 9747 } 9748 9749 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9750 assert(UseAVX > 2, "requires some form of EVEX"); 9751 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9752 attributes.set_rex_vex_w_reverted(); 9753 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9754 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9755 } 9756 9757 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9758 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r 9759 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9760 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9761 attributes.set_is_evex_instruction(); 9762 attributes.set_embedded_opmask_register_specifier(mask); 9763 if (merge) { 9764 attributes.reset_is_clear_context(); 9765 } 9766 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9767 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9768 } 9769 9770 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9771 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9772 InstructionMark im(this); 9773 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9774 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 9775 attributes.set_is_evex_instruction(); 9776 attributes.set_embedded_opmask_register_specifier(mask); 9777 if (merge) { 9778 attributes.reset_is_clear_context(); 9779 } 9780 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9781 emit_int8((unsigned char)0xEF); 9782 emit_operand(dst, src, 0); 9783 } 9784 9785 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9786 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r 9787 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9788 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9789 attributes.set_is_evex_instruction(); 9790 attributes.set_embedded_opmask_register_specifier(mask); 9791 if (merge) { 9792 attributes.reset_is_clear_context(); 9793 } 9794 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9795 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9796 } 9797 9798 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9799 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9800 InstructionMark im(this); 9801 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9802 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 9803 attributes.set_is_evex_instruction(); 9804 attributes.set_embedded_opmask_register_specifier(mask); 9805 if (merge) { 9806 attributes.reset_is_clear_context(); 9807 } 9808 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9809 emit_int8((unsigned char)0xEF); 9810 emit_operand(dst, src, 0); 9811 } 9812 9813 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9814 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9815 InstructionMark im(this); 9816 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9817 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 9818 attributes.set_is_evex_instruction(); 9819 attributes.set_embedded_opmask_register_specifier(mask); 9820 if (merge) { 9821 attributes.reset_is_clear_context(); 9822 } 9823 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9824 emit_int8((unsigned char)0xDB); 9825 emit_operand(dst, src, 0); 9826 } 9827 9828 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9829 assert(VM_Version::supports_evex(), "requires AVX512F"); 9830 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 9831 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9832 attributes.set_is_evex_instruction(); 9833 attributes.set_embedded_opmask_register_specifier(mask); 9834 if (merge) { 9835 attributes.reset_is_clear_context(); 9836 } 9837 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9838 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 9839 } 9840 9841 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9842 assert(VM_Version::supports_evex(), "requires AVX512F"); 9843 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 9844 InstructionMark im(this); 9845 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9846 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 9847 attributes.set_is_evex_instruction(); 9848 attributes.set_embedded_opmask_register_specifier(mask); 9849 if (merge) { 9850 attributes.reset_is_clear_context(); 9851 } 9852 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9853 emit_int8((unsigned char)0xDB); 9854 emit_operand(dst, src, 0); 9855 } 9856 9857 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9858 assert(VM_Version::supports_evex(), "requires AVX512F"); 9859 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 9860 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9861 attributes.set_is_evex_instruction(); 9862 attributes.set_embedded_opmask_register_specifier(mask); 9863 if (merge) { 9864 attributes.reset_is_clear_context(); 9865 } 9866 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9867 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 9868 } 9869 9870 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9871 assert(VM_Version::supports_evex(), "requires AVX512F"); 9872 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 9873 InstructionMark im(this); 9874 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9875 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 9876 attributes.set_is_evex_instruction(); 9877 attributes.set_embedded_opmask_register_specifier(mask); 9878 if (merge) { 9879 attributes.reset_is_clear_context(); 9880 } 9881 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9882 emit_int8((unsigned char)0xEB); 9883 emit_operand(dst, src, 0); 9884 } 9885 9886 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 9887 assert(VM_Version::supports_evex(), "requires EVEX support"); 9888 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9889 attributes.set_is_evex_instruction(); 9890 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9891 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 9892 } 9893 9894 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 9895 assert(VM_Version::supports_evex(), "requires EVEX support"); 9896 assert(dst != xnoreg, "sanity"); 9897 InstructionMark im(this); 9898 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9899 attributes.set_is_evex_instruction(); 9900 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 9901 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9902 emit_int8((unsigned char)0xEF); 9903 emit_operand(dst, src, 0); 9904 } 9905 9906 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9907 assert(VM_Version::supports_evex(), "requires EVEX support"); 9908 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9909 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9910 attributes.set_is_evex_instruction(); 9911 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9912 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9913 } 9914 9915 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9916 assert(VM_Version::supports_evex(), "requires EVEX support"); 9917 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9918 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9919 attributes.set_is_evex_instruction(); 9920 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9921 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9922 } 9923 9924 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9925 assert(VM_Version::supports_evex(), "requires EVEX support"); 9926 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9927 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9928 attributes.set_is_evex_instruction(); 9929 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9930 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9931 } 9932 9933 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 9934 assert(VM_Version::supports_evex(), "requires EVEX support"); 9935 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9936 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9937 attributes.set_is_evex_instruction(); 9938 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9939 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9940 } 9941 9942 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9943 assert(VM_Version::supports_evex(), "requires EVEX support"); 9944 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9945 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9946 attributes.set_is_evex_instruction(); 9947 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9948 emit_int16(0x15, (unsigned char)(0xC0 | encode)); 9949 } 9950 9951 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9952 assert(VM_Version::supports_evex(), "requires EVEX support"); 9953 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9954 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9955 attributes.set_is_evex_instruction(); 9956 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9957 emit_int16(0x15, (unsigned char)(0xC0 | encode)); 9958 } 9959 9960 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9961 assert(VM_Version::supports_evex(), "requires EVEX support"); 9962 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9963 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9964 attributes.set_is_evex_instruction(); 9965 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9966 emit_int16(0x14, (unsigned char)(0xC0 | encode)); 9967 } 9968 9969 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 9970 assert(VM_Version::supports_evex(), "requires EVEX support"); 9971 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 9972 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 9973 attributes.set_is_evex_instruction(); 9974 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9975 emit_int16(0x14, (unsigned char)(0xC0 | encode)); 9976 } 9977 9978 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9979 assert(VM_Version::supports_avx512cd(), ""); 9980 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9981 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9982 attributes.set_is_evex_instruction(); 9983 attributes.set_embedded_opmask_register_specifier(mask); 9984 if (merge) { 9985 attributes.reset_is_clear_context(); 9986 } 9987 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9988 emit_int16(0x44, (0xC0 | encode)); 9989 } 9990 9991 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9992 assert(VM_Version::supports_avx512cd(), ""); 9993 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9994 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9995 attributes.set_is_evex_instruction(); 9996 attributes.set_embedded_opmask_register_specifier(mask); 9997 if (merge) { 9998 attributes.reset_is_clear_context(); 9999 } 10000 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10001 emit_int16(0x44, (0xC0 | encode)); 10002 } 10003 10004 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) { 10005 assert(VM_Version::supports_evex(), "requires EVEX support"); 10006 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10007 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10008 attributes.set_is_evex_instruction(); 10009 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10010 emit_int8(0x25); 10011 emit_int8((unsigned char)(0xC0 | encode)); 10012 emit_int8(imm8); 10013 } 10014 10015 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) { 10016 assert(VM_Version::supports_evex(), "requires EVEX support"); 10017 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10018 assert(dst != xnoreg, "sanity"); 10019 InstructionMark im(this); 10020 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10021 attributes.set_is_evex_instruction(); 10022 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 10023 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10024 emit_int8(0x25); 10025 emit_operand(dst, src3, 1); 10026 emit_int8(imm8); 10027 } 10028 10029 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) { 10030 assert(VM_Version::supports_evex(), "requires AVX512F"); 10031 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 10032 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10033 attributes.set_is_evex_instruction(); 10034 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10035 emit_int8(0x25); 10036 emit_int8((unsigned char)(0xC0 | encode)); 10037 emit_int8(imm8); 10038 } 10039 10040 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) { 10041 assert(VM_Version::supports_evex(), "requires EVEX support"); 10042 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10043 assert(dst != xnoreg, "sanity"); 10044 InstructionMark im(this); 10045 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10046 attributes.set_is_evex_instruction(); 10047 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 10048 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10049 emit_int8(0x25); 10050 emit_operand(dst, src3, 1); 10051 emit_int8(imm8); 10052 } 10053 10054 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10055 assert(VM_Version::supports_evex(), ""); 10056 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10057 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10058 attributes.set_is_evex_instruction(); 10059 attributes.set_embedded_opmask_register_specifier(mask); 10060 if (merge) { 10061 attributes.reset_is_clear_context(); 10062 } 10063 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10064 emit_int16((unsigned char)0x88, (0xC0 | encode)); 10065 } 10066 10067 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10068 assert(VM_Version::supports_evex(), ""); 10069 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10070 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10071 attributes.set_is_evex_instruction(); 10072 attributes.set_embedded_opmask_register_specifier(mask); 10073 if (merge) { 10074 attributes.reset_is_clear_context(); 10075 } 10076 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10077 emit_int16((unsigned char)0x88, (0xC0 | encode)); 10078 } 10079 10080 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10081 assert(VM_Version::supports_avx512_vbmi2(), ""); 10082 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10083 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10084 attributes.set_is_evex_instruction(); 10085 attributes.set_embedded_opmask_register_specifier(mask); 10086 if (merge) { 10087 attributes.reset_is_clear_context(); 10088 } 10089 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10090 emit_int16(0x62, (0xC0 | encode)); 10091 } 10092 10093 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10094 assert(VM_Version::supports_avx512_vbmi2(), ""); 10095 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10096 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10097 attributes.set_is_evex_instruction(); 10098 attributes.set_embedded_opmask_register_specifier(mask); 10099 if (merge) { 10100 attributes.reset_is_clear_context(); 10101 } 10102 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10103 emit_int16(0x62, (0xC0 | encode)); 10104 } 10105 10106 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10107 assert(VM_Version::supports_evex(), ""); 10108 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10109 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10110 attributes.set_is_evex_instruction(); 10111 attributes.set_embedded_opmask_register_specifier(mask); 10112 if (merge) { 10113 attributes.reset_is_clear_context(); 10114 } 10115 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10116 emit_int16((unsigned char)0x89, (0xC0 | encode)); 10117 } 10118 10119 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 10120 assert(VM_Version::supports_evex(), ""); 10121 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10122 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10123 attributes.set_is_evex_instruction(); 10124 attributes.set_embedded_opmask_register_specifier(mask); 10125 if (merge) { 10126 attributes.reset_is_clear_context(); 10127 } 10128 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10129 emit_int16((unsigned char)0x89, (0xC0 | encode)); 10130 } 10131 10132 // vinserti forms 10133 10134 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10135 assert(VM_Version::supports_avx2(), ""); 10136 assert(imm8 <= 0x01, "imm8: %u", imm8); 10137 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10138 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10139 // last byte: 10140 // 0x00 - insert into lower 128 bits 10141 // 0x01 - insert into upper 128 bits 10142 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01); 10143 } 10144 10145 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 10146 assert(VM_Version::supports_avx2(), ""); 10147 assert(dst != xnoreg, "sanity"); 10148 assert(imm8 <= 0x01, "imm8: %u", imm8); 10149 InstructionMark im(this); 10150 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10151 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10152 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10153 emit_int8(0x38); 10154 emit_operand(dst, src, 1); 10155 // 0x00 - insert into lower 128 bits 10156 // 0x01 - insert into upper 128 bits 10157 emit_int8(imm8 & 0x01); 10158 } 10159 10160 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10161 assert(VM_Version::supports_evex(), ""); 10162 assert(imm8 <= 0x03, "imm8: %u", imm8); 10163 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10164 attributes.set_is_evex_instruction(); 10165 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10166 // imm8: 10167 // 0x00 - insert into q0 128 bits (0..127) 10168 // 0x01 - insert into q1 128 bits (128..255) 10169 // 0x02 - insert into q2 128 bits (256..383) 10170 // 0x03 - insert into q3 128 bits (384..511) 10171 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03); 10172 } 10173 10174 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 10175 assert(VM_Version::supports_evex(), ""); 10176 assert(dst != xnoreg, "sanity"); 10177 assert(imm8 <= 0x03, "imm8: %u", imm8); 10178 InstructionMark im(this); 10179 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10180 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10181 attributes.set_is_evex_instruction(); 10182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10183 emit_int8(0x18); 10184 emit_operand(dst, src, 1); 10185 // 0x00 - insert into q0 128 bits (0..127) 10186 // 0x01 - insert into q1 128 bits (128..255) 10187 // 0x02 - insert into q2 128 bits (256..383) 10188 // 0x03 - insert into q3 128 bits (384..511) 10189 emit_int8(imm8 & 0x03); 10190 } 10191 10192 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10193 assert(VM_Version::supports_evex(), ""); 10194 assert(imm8 <= 0x01, "imm8: %u", imm8); 10195 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10196 attributes.set_is_evex_instruction(); 10197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10198 //imm8: 10199 // 0x00 - insert into lower 256 bits 10200 // 0x01 - insert into upper 256 bits 10201 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01); 10202 } 10203 10204 void Assembler::evinserti64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8, int vector_len) { 10205 assert(VM_Version::supports_avx512dq(), ""); 10206 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10207 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10208 attributes.set_is_evex_instruction(); 10209 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10210 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03); 10211 } 10212 10213 10214 // vinsertf forms 10215 10216 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10217 assert(VM_Version::supports_avx(), ""); 10218 assert(imm8 <= 0x01, "imm8: %u", imm8); 10219 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10220 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10221 // imm8: 10222 // 0x00 - insert into lower 128 bits 10223 // 0x01 - insert into upper 128 bits 10224 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01); 10225 } 10226 10227 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 10228 assert(VM_Version::supports_avx(), ""); 10229 assert(dst != xnoreg, "sanity"); 10230 assert(imm8 <= 0x01, "imm8: %u", imm8); 10231 InstructionMark im(this); 10232 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10233 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10234 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10235 emit_int8(0x18); 10236 emit_operand(dst, src, 1); 10237 // 0x00 - insert into lower 128 bits 10238 // 0x01 - insert into upper 128 bits 10239 emit_int8(imm8 & 0x01); 10240 } 10241 10242 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10243 assert(VM_Version::supports_evex(), ""); 10244 assert(imm8 <= 0x03, "imm8: %u", imm8); 10245 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10247 // imm8: 10248 // 0x00 - insert into q0 128 bits (0..127) 10249 // 0x01 - insert into q1 128 bits (128..255) 10250 // 0x02 - insert into q0 128 bits (256..383) 10251 // 0x03 - insert into q1 128 bits (384..512) 10252 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03); 10253 } 10254 10255 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 10256 assert(VM_Version::supports_evex(), ""); 10257 assert(dst != xnoreg, "sanity"); 10258 assert(imm8 <= 0x03, "imm8: %u", imm8); 10259 InstructionMark im(this); 10260 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10261 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10262 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10263 emit_int8(0x18); 10264 emit_operand(dst, src, 1); 10265 // 0x00 - insert into q0 128 bits (0..127) 10266 // 0x01 - insert into q1 128 bits (128..255) 10267 // 0x02 - insert into q0 128 bits (256..383) 10268 // 0x03 - insert into q1 128 bits (384..512) 10269 emit_int8(imm8 & 0x03); 10270 } 10271 10272 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 10273 assert(VM_Version::supports_evex(), ""); 10274 assert(imm8 <= 0x01, "imm8: %u", imm8); 10275 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10276 attributes.set_is_evex_instruction(); 10277 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10278 // imm8: 10279 // 0x00 - insert into lower 256 bits 10280 // 0x01 - insert into upper 256 bits 10281 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01); 10282 } 10283 10284 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 10285 assert(VM_Version::supports_evex(), ""); 10286 assert(dst != xnoreg, "sanity"); 10287 assert(imm8 <= 0x01, "imm8: %u", imm8); 10288 InstructionMark im(this); 10289 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10290 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit); 10291 attributes.set_is_evex_instruction(); 10292 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10293 emit_int8(0x1A); 10294 emit_operand(dst, src, 1); 10295 // 0x00 - insert into lower 256 bits 10296 // 0x01 - insert into upper 256 bits 10297 emit_int8(imm8 & 0x01); 10298 } 10299 10300 10301 // vextracti forms 10302 10303 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10304 assert(VM_Version::supports_avx2(), ""); 10305 assert(imm8 <= 0x01, "imm8: %u", imm8); 10306 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10307 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10308 // imm8: 10309 // 0x00 - extract from lower 128 bits 10310 // 0x01 - extract from upper 128 bits 10311 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01); 10312 } 10313 10314 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) { 10315 assert(VM_Version::supports_avx2(), ""); 10316 assert(src != xnoreg, "sanity"); 10317 assert(imm8 <= 0x01, "imm8: %u", imm8); 10318 InstructionMark im(this); 10319 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10320 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10321 attributes.reset_is_clear_context(); 10322 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10323 emit_int8(0x39); 10324 emit_operand(src, dst, 1); 10325 // 0x00 - extract from lower 128 bits 10326 // 0x01 - extract from upper 128 bits 10327 emit_int8(imm8 & 0x01); 10328 } 10329 10330 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10331 assert(VM_Version::supports_evex(), ""); 10332 assert(imm8 <= 0x03, "imm8: %u", imm8); 10333 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10334 attributes.set_is_evex_instruction(); 10335 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10336 // imm8: 10337 // 0x00 - extract from bits 127:0 10338 // 0x01 - extract from bits 255:128 10339 // 0x02 - extract from bits 383:256 10340 // 0x03 - extract from bits 511:384 10341 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03); 10342 } 10343 10344 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) { 10345 assert(VM_Version::supports_evex(), ""); 10346 assert(src != xnoreg, "sanity"); 10347 assert(imm8 <= 0x03, "imm8: %u", imm8); 10348 InstructionMark im(this); 10349 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10350 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10351 attributes.reset_is_clear_context(); 10352 attributes.set_is_evex_instruction(); 10353 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10354 emit_int8(0x39); 10355 emit_operand(src, dst, 1); 10356 // 0x00 - extract from bits 127:0 10357 // 0x01 - extract from bits 255:128 10358 // 0x02 - extract from bits 383:256 10359 // 0x03 - extract from bits 511:384 10360 emit_int8(imm8 & 0x03); 10361 } 10362 10363 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10364 assert(VM_Version::supports_avx512dq(), ""); 10365 assert(imm8 <= 0x03, "imm8: %u", imm8); 10366 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10367 attributes.set_is_evex_instruction(); 10368 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10369 // imm8: 10370 // 0x00 - extract from bits 127:0 10371 // 0x01 - extract from bits 255:128 10372 // 0x02 - extract from bits 383:256 10373 // 0x03 - extract from bits 511:384 10374 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03); 10375 } 10376 10377 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10378 assert(VM_Version::supports_evex(), ""); 10379 assert(imm8 <= 0x01, "imm8: %u", imm8); 10380 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10381 attributes.set_is_evex_instruction(); 10382 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10383 // imm8: 10384 // 0x00 - extract from lower 256 bits 10385 // 0x01 - extract from upper 256 bits 10386 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01); 10387 } 10388 10389 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) { 10390 assert(VM_Version::supports_evex(), ""); 10391 assert(src != xnoreg, "sanity"); 10392 assert(imm8 <= 0x01, "imm8: %u", imm8); 10393 InstructionMark im(this); 10394 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10395 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit); 10396 attributes.reset_is_clear_context(); 10397 attributes.set_is_evex_instruction(); 10398 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10399 emit_int8(0x38); 10400 emit_operand(src, dst, 1); 10401 // 0x00 - extract from lower 256 bits 10402 // 0x01 - extract from upper 256 bits 10403 emit_int8(imm8 & 0x01); 10404 } 10405 // vextractf forms 10406 10407 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10408 assert(VM_Version::supports_avx(), ""); 10409 assert(imm8 <= 0x01, "imm8: %u", imm8); 10410 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10411 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10412 // imm8: 10413 // 0x00 - extract from lower 128 bits 10414 // 0x01 - extract from upper 128 bits 10415 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01); 10416 } 10417 10418 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) { 10419 assert(VM_Version::supports_avx(), ""); 10420 assert(src != xnoreg, "sanity"); 10421 assert(imm8 <= 0x01, "imm8: %u", imm8); 10422 InstructionMark im(this); 10423 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10424 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10425 attributes.reset_is_clear_context(); 10426 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10427 emit_int8(0x19); 10428 emit_operand(src, dst, 1); 10429 // 0x00 - extract from lower 128 bits 10430 // 0x01 - extract from upper 128 bits 10431 emit_int8(imm8 & 0x01); 10432 } 10433 10434 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10435 assert(VM_Version::supports_evex(), ""); 10436 assert(imm8 <= 0x03, "imm8: %u", imm8); 10437 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10438 attributes.set_is_evex_instruction(); 10439 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10440 // imm8: 10441 // 0x00 - extract from bits 127:0 10442 // 0x01 - extract from bits 255:128 10443 // 0x02 - extract from bits 383:256 10444 // 0x03 - extract from bits 511:384 10445 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03); 10446 } 10447 10448 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) { 10449 assert(VM_Version::supports_evex(), ""); 10450 assert(src != xnoreg, "sanity"); 10451 assert(imm8 <= 0x03, "imm8: %u", imm8); 10452 InstructionMark im(this); 10453 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10454 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10455 attributes.reset_is_clear_context(); 10456 attributes.set_is_evex_instruction(); 10457 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10458 emit_int8(0x19); 10459 emit_operand(src, dst, 1); 10460 // 0x00 - extract from bits 127:0 10461 // 0x01 - extract from bits 255:128 10462 // 0x02 - extract from bits 383:256 10463 // 0x03 - extract from bits 511:384 10464 emit_int8(imm8 & 0x03); 10465 } 10466 10467 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10468 assert(VM_Version::supports_avx512dq(), ""); 10469 assert(imm8 <= 0x03, "imm8: %u", imm8); 10470 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10471 attributes.set_is_evex_instruction(); 10472 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10473 // imm8: 10474 // 0x00 - extract from bits 127:0 10475 // 0x01 - extract from bits 255:128 10476 // 0x02 - extract from bits 383:256 10477 // 0x03 - extract from bits 511:384 10478 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03); 10479 } 10480 10481 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 10482 assert(VM_Version::supports_evex(), ""); 10483 assert(imm8 <= 0x01, "imm8: %u", imm8); 10484 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10485 attributes.set_is_evex_instruction(); 10486 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10487 // imm8: 10488 // 0x00 - extract from lower 256 bits 10489 // 0x01 - extract from upper 256 bits 10490 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01); 10491 } 10492 10493 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) { 10494 assert(VM_Version::supports_evex(), ""); 10495 assert(src != xnoreg, "sanity"); 10496 assert(imm8 <= 0x01, "imm8: %u", imm8); 10497 InstructionMark im(this); 10498 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10499 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit); 10500 attributes.reset_is_clear_context(); 10501 attributes.set_is_evex_instruction(); 10502 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10503 emit_int8(0x1B); 10504 emit_operand(src, dst, 1); 10505 // 0x00 - extract from lower 256 bits 10506 // 0x01 - extract from upper 256 bits 10507 emit_int8(imm8 & 0x01); 10508 } 10509 10510 void Assembler::extractps(Register dst, XMMRegister src, uint8_t imm8) { 10511 assert(VM_Version::supports_sse4_1(), ""); 10512 assert(imm8 <= 0x03, "imm8: %u", imm8); 10513 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 10514 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true); 10515 // imm8: 10516 // 0x00 - extract from bits 31:0 10517 // 0x01 - extract from bits 63:32 10518 // 0x02 - extract from bits 95:64 10519 // 0x03 - extract from bits 127:96 10520 emit_int24(0x17, (0xC0 | encode), imm8 & 0x03); 10521 } 10522 10523 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 10524 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) { 10525 assert(VM_Version::supports_avx2(), ""); 10526 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10527 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10528 emit_int16(0x78, (0xC0 | encode)); 10529 } 10530 10531 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) { 10532 assert(VM_Version::supports_avx2(), ""); 10533 assert(dst != xnoreg, "sanity"); 10534 InstructionMark im(this); 10535 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10536 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 10537 // swap src<->dst for encoding 10538 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10539 emit_int8(0x78); 10540 emit_operand(dst, src, 0); 10541 } 10542 10543 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 10544 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 10545 assert(VM_Version::supports_avx2(), ""); 10546 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10547 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10548 emit_int16(0x79, (0xC0 | encode)); 10549 } 10550 10551 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) { 10552 assert(VM_Version::supports_avx2(), ""); 10553 assert(dst != xnoreg, "sanity"); 10554 InstructionMark im(this); 10555 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10556 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 10557 // swap src<->dst for encoding 10558 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10559 emit_int8(0x79); 10560 emit_operand(dst, src, 0); 10561 } 10562 10563 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10564 assert(UseAVX > 0, "requires some form of AVX"); 10565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10566 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10567 emit_int16((unsigned char)0xF6, (0xC0 | encode)); 10568 } 10569 10570 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10571 assert(UseAVX > 0, "requires some form of AVX"); 10572 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10573 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10574 emit_int16(0x69, (0xC0 | encode)); 10575 } 10576 10577 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10578 assert(UseAVX > 0, "requires some form of AVX"); 10579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10580 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10581 emit_int16(0x61, (0xC0 | encode)); 10582 } 10583 10584 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10585 assert(UseAVX > 0, "requires some form of AVX"); 10586 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10587 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10588 emit_int16(0x6A, (0xC0 | encode)); 10589 } 10590 10591 void Assembler::vpunpckhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10592 assert(UseAVX > 0, "requires some form of AVX"); 10593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10594 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10595 emit_int16(0x6D, (0xC0 | encode)); 10596 } 10597 10598 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10599 assert(UseAVX > 0, "requires some form of AVX"); 10600 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10601 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10602 emit_int16(0x62, (0xC0 | encode)); 10603 } 10604 10605 void Assembler::vpunpcklqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 10606 assert(UseAVX > 0, "requires some form of AVX"); 10607 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10608 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10609 emit_int16(0x6C, (0xC0 | encode)); 10610 } 10611 10612 // xmm/mem sourced byte/word/dword/qword replicate 10613 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10614 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10615 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10616 attributes.set_is_evex_instruction(); 10617 attributes.set_embedded_opmask_register_specifier(mask); 10618 if (merge) { 10619 attributes.reset_is_clear_context(); 10620 } 10621 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10622 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 10623 } 10624 10625 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10626 InstructionMark im(this); 10627 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10628 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10629 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10630 attributes.set_is_evex_instruction(); 10631 attributes.set_embedded_opmask_register_specifier(mask); 10632 if (merge) { 10633 attributes.reset_is_clear_context(); 10634 } 10635 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10636 emit_int8((unsigned char)0xFC); 10637 emit_operand(dst, src, 0); 10638 } 10639 10640 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10641 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10642 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10643 attributes.set_is_evex_instruction(); 10644 attributes.set_embedded_opmask_register_specifier(mask); 10645 if (merge) { 10646 attributes.reset_is_clear_context(); 10647 } 10648 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10649 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 10650 } 10651 10652 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10653 InstructionMark im(this); 10654 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10655 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10656 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10657 attributes.set_is_evex_instruction(); 10658 attributes.set_embedded_opmask_register_specifier(mask); 10659 if (merge) { 10660 attributes.reset_is_clear_context(); 10661 } 10662 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10663 emit_int8((unsigned char)0xFD); 10664 emit_operand(dst, src, 0); 10665 } 10666 10667 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10668 assert(VM_Version::supports_evex(), ""); 10669 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10670 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10671 attributes.set_is_evex_instruction(); 10672 attributes.set_embedded_opmask_register_specifier(mask); 10673 if (merge) { 10674 attributes.reset_is_clear_context(); 10675 } 10676 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10677 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 10678 } 10679 10680 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10681 InstructionMark im(this); 10682 assert(VM_Version::supports_evex(), ""); 10683 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10684 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10685 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10686 attributes.set_is_evex_instruction(); 10687 attributes.set_embedded_opmask_register_specifier(mask); 10688 if (merge) { 10689 attributes.reset_is_clear_context(); 10690 } 10691 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10692 emit_int8((unsigned char)0xFE); 10693 emit_operand(dst, src, 0); 10694 } 10695 10696 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10697 assert(VM_Version::supports_evex(), ""); 10698 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10699 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10700 attributes.set_is_evex_instruction(); 10701 attributes.set_embedded_opmask_register_specifier(mask); 10702 if (merge) { 10703 attributes.reset_is_clear_context(); 10704 } 10705 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10706 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 10707 } 10708 10709 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10710 InstructionMark im(this); 10711 assert(VM_Version::supports_evex(), ""); 10712 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10713 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10714 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10715 attributes.set_is_evex_instruction(); 10716 attributes.set_embedded_opmask_register_specifier(mask); 10717 if (merge) { 10718 attributes.reset_is_clear_context(); 10719 } 10720 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10721 emit_int8((unsigned char)0xD4); 10722 emit_operand(dst, src, 0); 10723 } 10724 10725 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10726 assert(VM_Version::supports_evex(), ""); 10727 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10728 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10729 attributes.set_is_evex_instruction(); 10730 attributes.set_embedded_opmask_register_specifier(mask); 10731 if (merge) { 10732 attributes.reset_is_clear_context(); 10733 } 10734 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 10735 emit_int16(0x58, (0xC0 | encode)); 10736 } 10737 10738 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10739 InstructionMark im(this); 10740 assert(VM_Version::supports_evex(), ""); 10741 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10742 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10744 attributes.set_is_evex_instruction(); 10745 attributes.set_embedded_opmask_register_specifier(mask); 10746 if (merge) { 10747 attributes.reset_is_clear_context(); 10748 } 10749 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 10750 emit_int8(0x58); 10751 emit_operand(dst, src, 0); 10752 } 10753 10754 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10755 assert(VM_Version::supports_evex(), ""); 10756 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10757 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10758 attributes.set_is_evex_instruction(); 10759 attributes.set_embedded_opmask_register_specifier(mask); 10760 if (merge) { 10761 attributes.reset_is_clear_context(); 10762 } 10763 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10764 emit_int16(0x58, (0xC0 | encode)); 10765 } 10766 10767 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10768 InstructionMark im(this); 10769 assert(VM_Version::supports_evex(), ""); 10770 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10771 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10772 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10773 attributes.set_is_evex_instruction(); 10774 attributes.set_embedded_opmask_register_specifier(mask); 10775 if (merge) { 10776 attributes.reset_is_clear_context(); 10777 } 10778 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10779 emit_int8(0x58); 10780 emit_operand(dst, src, 0); 10781 } 10782 10783 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10784 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10785 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10786 attributes.set_is_evex_instruction(); 10787 attributes.set_embedded_opmask_register_specifier(mask); 10788 if (merge) { 10789 attributes.reset_is_clear_context(); 10790 } 10791 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10792 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 10793 } 10794 10795 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10796 InstructionMark im(this); 10797 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10798 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10799 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10800 attributes.set_is_evex_instruction(); 10801 attributes.set_embedded_opmask_register_specifier(mask); 10802 if (merge) { 10803 attributes.reset_is_clear_context(); 10804 } 10805 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10806 emit_int8((unsigned char)0xF8); 10807 emit_operand(dst, src, 0); 10808 } 10809 10810 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10811 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10812 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10813 attributes.set_is_evex_instruction(); 10814 attributes.set_embedded_opmask_register_specifier(mask); 10815 if (merge) { 10816 attributes.reset_is_clear_context(); 10817 } 10818 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10819 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 10820 } 10821 10822 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10823 InstructionMark im(this); 10824 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10825 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10826 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10827 attributes.set_is_evex_instruction(); 10828 attributes.set_embedded_opmask_register_specifier(mask); 10829 if (merge) { 10830 attributes.reset_is_clear_context(); 10831 } 10832 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10833 emit_int8((unsigned char)0xF9); 10834 emit_operand(dst, src, 0); 10835 } 10836 10837 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10838 assert(VM_Version::supports_evex(), ""); 10839 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10840 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10841 attributes.set_is_evex_instruction(); 10842 attributes.set_embedded_opmask_register_specifier(mask); 10843 if (merge) { 10844 attributes.reset_is_clear_context(); 10845 } 10846 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10847 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 10848 } 10849 10850 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10851 InstructionMark im(this); 10852 assert(VM_Version::supports_evex(), ""); 10853 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10854 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10855 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10856 attributes.set_is_evex_instruction(); 10857 attributes.set_embedded_opmask_register_specifier(mask); 10858 if (merge) { 10859 attributes.reset_is_clear_context(); 10860 } 10861 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10862 emit_int8((unsigned char)0xFA); 10863 emit_operand(dst, src, 0); 10864 } 10865 10866 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10867 assert(VM_Version::supports_evex(), ""); 10868 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10869 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10870 attributes.set_is_evex_instruction(); 10871 attributes.set_embedded_opmask_register_specifier(mask); 10872 if (merge) { 10873 attributes.reset_is_clear_context(); 10874 } 10875 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10876 emit_int16((unsigned char)0xFB, (0xC0 | encode)); 10877 } 10878 10879 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10880 InstructionMark im(this); 10881 assert(VM_Version::supports_evex(), ""); 10882 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10883 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10884 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10885 attributes.set_is_evex_instruction(); 10886 attributes.set_embedded_opmask_register_specifier(mask); 10887 if (merge) { 10888 attributes.reset_is_clear_context(); 10889 } 10890 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10891 emit_int8((unsigned char)0xFB); 10892 emit_operand(dst, src, 0); 10893 } 10894 10895 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10896 assert(VM_Version::supports_evex(), ""); 10897 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10898 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10899 attributes.set_is_evex_instruction(); 10900 attributes.set_embedded_opmask_register_specifier(mask); 10901 if (merge) { 10902 attributes.reset_is_clear_context(); 10903 } 10904 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 10905 emit_int16(0x5C, (0xC0 | encode)); 10906 } 10907 10908 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10909 InstructionMark im(this); 10910 assert(VM_Version::supports_evex(), ""); 10911 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10912 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10913 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10914 attributes.set_is_evex_instruction(); 10915 attributes.set_embedded_opmask_register_specifier(mask); 10916 if (merge) { 10917 attributes.reset_is_clear_context(); 10918 } 10919 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 10920 emit_int8(0x5C); 10921 emit_operand(dst, src, 0); 10922 } 10923 10924 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10925 assert(VM_Version::supports_evex(), ""); 10926 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10927 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10928 attributes.set_is_evex_instruction(); 10929 attributes.set_embedded_opmask_register_specifier(mask); 10930 if (merge) { 10931 attributes.reset_is_clear_context(); 10932 } 10933 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10934 emit_int16(0x5C, (0xC0 | encode)); 10935 } 10936 10937 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10938 InstructionMark im(this); 10939 assert(VM_Version::supports_evex(), ""); 10940 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10941 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 10943 attributes.set_is_evex_instruction(); 10944 attributes.set_embedded_opmask_register_specifier(mask); 10945 if (merge) { 10946 attributes.reset_is_clear_context(); 10947 } 10948 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10949 emit_int8(0x5C); 10950 emit_operand(dst, src, 0); 10951 } 10952 10953 void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10954 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10955 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10956 attributes.set_is_evex_instruction(); 10957 attributes.set_embedded_opmask_register_specifier(mask); 10958 if (merge) { 10959 attributes.reset_is_clear_context(); 10960 } 10961 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10962 emit_int16((unsigned char)0xEC, (0xC0 | encode)); 10963 } 10964 10965 void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10966 InstructionMark im(this); 10967 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10968 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10969 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10970 attributes.set_is_evex_instruction(); 10971 attributes.set_embedded_opmask_register_specifier(mask); 10972 if (merge) { 10973 attributes.reset_is_clear_context(); 10974 } 10975 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10976 emit_int8((unsigned char)0xEC); 10977 emit_operand(dst, src, 0); 10978 } 10979 10980 void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10981 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10982 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10983 attributes.set_is_evex_instruction(); 10984 attributes.set_embedded_opmask_register_specifier(mask); 10985 if (merge) { 10986 attributes.reset_is_clear_context(); 10987 } 10988 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10989 emit_int16((unsigned char)0xED, (0xC0 | encode)); 10990 } 10991 10992 void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10993 InstructionMark im(this); 10994 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10995 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 10996 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 10997 attributes.set_is_evex_instruction(); 10998 attributes.set_embedded_opmask_register_specifier(mask); 10999 if (merge) { 11000 attributes.reset_is_clear_context(); 11001 } 11002 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11003 emit_int8((unsigned char)0xED); 11004 emit_operand(dst, src, 0); 11005 } 11006 11007 void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11008 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11009 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11010 attributes.set_is_evex_instruction(); 11011 attributes.set_embedded_opmask_register_specifier(mask); 11012 if (merge) { 11013 attributes.reset_is_clear_context(); 11014 } 11015 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11016 emit_int16((unsigned char)0xDC, (0xC0 | encode)); 11017 } 11018 11019 void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11020 InstructionMark im(this); 11021 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11022 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11023 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11024 attributes.set_is_evex_instruction(); 11025 attributes.set_embedded_opmask_register_specifier(mask); 11026 if (merge) { 11027 attributes.reset_is_clear_context(); 11028 } 11029 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11030 emit_int8((unsigned char)0xDC); 11031 emit_operand(dst, src, 0); 11032 } 11033 11034 void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11035 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11036 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11037 attributes.set_is_evex_instruction(); 11038 attributes.set_embedded_opmask_register_specifier(mask); 11039 if (merge) { 11040 attributes.reset_is_clear_context(); 11041 } 11042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11043 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 11044 } 11045 11046 void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11047 InstructionMark im(this); 11048 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11049 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11050 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11051 attributes.set_is_evex_instruction(); 11052 attributes.set_embedded_opmask_register_specifier(mask); 11053 if (merge) { 11054 attributes.reset_is_clear_context(); 11055 } 11056 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11057 emit_int8((unsigned char)0xDD); 11058 emit_operand(dst, src, 0); 11059 } 11060 11061 void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11062 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11063 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11064 attributes.set_is_evex_instruction(); 11065 attributes.set_embedded_opmask_register_specifier(mask); 11066 if (merge) { 11067 attributes.reset_is_clear_context(); 11068 } 11069 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11070 emit_int16((unsigned char)0xE8, (0xC0 | encode)); 11071 } 11072 11073 void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11074 InstructionMark im(this); 11075 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11076 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11077 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11078 attributes.set_is_evex_instruction(); 11079 attributes.set_embedded_opmask_register_specifier(mask); 11080 if (merge) { 11081 attributes.reset_is_clear_context(); 11082 } 11083 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11084 emit_int8((unsigned char)0xE8); 11085 emit_operand(dst, src, 0); 11086 } 11087 11088 void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11089 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11090 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11091 attributes.set_is_evex_instruction(); 11092 attributes.set_embedded_opmask_register_specifier(mask); 11093 if (merge) { 11094 attributes.reset_is_clear_context(); 11095 } 11096 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11097 emit_int16((unsigned char)0xE9, (0xC0 | encode)); 11098 } 11099 11100 void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11101 InstructionMark im(this); 11102 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11103 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11104 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11105 attributes.set_is_evex_instruction(); 11106 attributes.set_embedded_opmask_register_specifier(mask); 11107 if (merge) { 11108 attributes.reset_is_clear_context(); 11109 } 11110 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11111 emit_int8((unsigned char)0xE9); 11112 emit_operand(dst, src, 0); 11113 } 11114 11115 void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11116 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11118 attributes.set_is_evex_instruction(); 11119 attributes.set_embedded_opmask_register_specifier(mask); 11120 if (merge) { 11121 attributes.reset_is_clear_context(); 11122 } 11123 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11124 emit_int16((unsigned char)0xD8, (0xC0 | encode)); 11125 } 11126 11127 void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11128 InstructionMark im(this); 11129 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11130 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11131 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11132 attributes.set_is_evex_instruction(); 11133 attributes.set_embedded_opmask_register_specifier(mask); 11134 if (merge) { 11135 attributes.reset_is_clear_context(); 11136 } 11137 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11138 emit_int8((unsigned char)0xD8); 11139 emit_operand(dst, src, 0); 11140 } 11141 11142 void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11143 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11144 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11145 attributes.set_is_evex_instruction(); 11146 attributes.set_embedded_opmask_register_specifier(mask); 11147 if (merge) { 11148 attributes.reset_is_clear_context(); 11149 } 11150 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11151 emit_int16((unsigned char)0xD9, (0xC0 | encode)); 11152 } 11153 11154 11155 void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11156 InstructionMark im(this); 11157 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11158 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11159 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11160 attributes.set_is_evex_instruction(); 11161 attributes.set_embedded_opmask_register_specifier(mask); 11162 if (merge) { 11163 attributes.reset_is_clear_context(); 11164 } 11165 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11166 emit_int8((unsigned char)0xD9); 11167 emit_operand(dst, src, 0); 11168 } 11169 11170 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11171 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11172 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11173 attributes.set_is_evex_instruction(); 11174 attributes.set_embedded_opmask_register_specifier(mask); 11175 if (merge) { 11176 attributes.reset_is_clear_context(); 11177 } 11178 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11179 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 11180 } 11181 11182 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11183 InstructionMark im(this); 11184 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11185 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11186 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11187 attributes.set_is_evex_instruction(); 11188 attributes.set_embedded_opmask_register_specifier(mask); 11189 if (merge) { 11190 attributes.reset_is_clear_context(); 11191 } 11192 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11193 emit_int8((unsigned char)0xD5); 11194 emit_operand(dst, src, 0); 11195 } 11196 11197 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11198 assert(VM_Version::supports_evex(), ""); 11199 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11200 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11201 attributes.set_is_evex_instruction(); 11202 attributes.set_embedded_opmask_register_specifier(mask); 11203 if (merge) { 11204 attributes.reset_is_clear_context(); 11205 } 11206 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11207 emit_int16(0x40, (0xC0 | encode)); 11208 } 11209 11210 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11211 InstructionMark im(this); 11212 assert(VM_Version::supports_evex(), ""); 11213 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11214 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11215 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11216 attributes.set_is_evex_instruction(); 11217 attributes.set_embedded_opmask_register_specifier(mask); 11218 if (merge) { 11219 attributes.reset_is_clear_context(); 11220 } 11221 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11222 emit_int8(0x40); 11223 emit_operand(dst, src, 0); 11224 } 11225 11226 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11227 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11228 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11229 attributes.set_is_evex_instruction(); 11230 attributes.set_embedded_opmask_register_specifier(mask); 11231 if (merge) { 11232 attributes.reset_is_clear_context(); 11233 } 11234 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11235 emit_int16(0x40, (0xC0 | encode)); 11236 } 11237 11238 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11239 InstructionMark im(this); 11240 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11241 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11242 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11243 attributes.set_is_evex_instruction(); 11244 attributes.set_embedded_opmask_register_specifier(mask); 11245 if (merge) { 11246 attributes.reset_is_clear_context(); 11247 } 11248 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11249 emit_int8(0x40); 11250 emit_operand(dst, src, 0); 11251 } 11252 11253 void Assembler::evpmulhw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11254 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11255 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11256 attributes.set_is_evex_instruction(); 11257 attributes.set_embedded_opmask_register_specifier(mask); 11258 if (merge) { 11259 attributes.reset_is_clear_context(); 11260 } 11261 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11262 emit_int16((unsigned char)0xE5, (0xC0 | encode)); 11263 } 11264 11265 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11266 assert(VM_Version::supports_evex(), ""); 11267 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11268 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11269 attributes.set_is_evex_instruction(); 11270 attributes.set_embedded_opmask_register_specifier(mask); 11271 if (merge) { 11272 attributes.reset_is_clear_context(); 11273 } 11274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11275 emit_int16(0x59, (0xC0 | encode)); 11276 } 11277 11278 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11279 InstructionMark im(this); 11280 assert(VM_Version::supports_evex(), ""); 11281 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11282 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11283 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11284 attributes.set_is_evex_instruction(); 11285 attributes.set_embedded_opmask_register_specifier(mask); 11286 if (merge) { 11287 attributes.reset_is_clear_context(); 11288 } 11289 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11290 emit_int8(0x59); 11291 emit_operand(dst, src, 0); 11292 } 11293 11294 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11295 assert(VM_Version::supports_evex(), ""); 11296 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11297 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11298 attributes.set_is_evex_instruction(); 11299 attributes.set_embedded_opmask_register_specifier(mask); 11300 if (merge) { 11301 attributes.reset_is_clear_context(); 11302 } 11303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11304 emit_int16(0x59, (0xC0 | encode)); 11305 } 11306 11307 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11308 InstructionMark im(this); 11309 assert(VM_Version::supports_evex(), ""); 11310 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11311 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11312 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11313 attributes.set_is_evex_instruction(); 11314 attributes.set_embedded_opmask_register_specifier(mask); 11315 if (merge) { 11316 attributes.reset_is_clear_context(); 11317 } 11318 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11319 emit_int8(0x59); 11320 emit_operand(dst, src, 0); 11321 } 11322 11323 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11324 assert(VM_Version::supports_evex(), ""); 11325 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11326 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11327 attributes.set_is_evex_instruction(); 11328 attributes.set_embedded_opmask_register_specifier(mask); 11329 if (merge) { 11330 attributes.reset_is_clear_context(); 11331 } 11332 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11333 emit_int16(0x51, (0xC0 | encode)); 11334 } 11335 11336 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11337 InstructionMark im(this); 11338 assert(VM_Version::supports_evex(), ""); 11339 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11340 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11341 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11342 attributes.set_is_evex_instruction(); 11343 attributes.set_embedded_opmask_register_specifier(mask); 11344 if (merge) { 11345 attributes.reset_is_clear_context(); 11346 } 11347 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11348 emit_int8(0x51); 11349 emit_operand(dst, src, 0); 11350 } 11351 11352 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11353 assert(VM_Version::supports_evex(), ""); 11354 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11355 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11356 attributes.set_is_evex_instruction(); 11357 attributes.set_embedded_opmask_register_specifier(mask); 11358 if (merge) { 11359 attributes.reset_is_clear_context(); 11360 } 11361 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11362 emit_int16(0x51, (0xC0 | encode)); 11363 } 11364 11365 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11366 InstructionMark im(this); 11367 assert(VM_Version::supports_evex(), ""); 11368 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11369 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11370 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11371 attributes.set_is_evex_instruction(); 11372 attributes.set_embedded_opmask_register_specifier(mask); 11373 if (merge) { 11374 attributes.reset_is_clear_context(); 11375 } 11376 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11377 emit_int8(0x51); 11378 emit_operand(dst, src, 0); 11379 } 11380 11381 11382 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11383 assert(VM_Version::supports_evex(), ""); 11384 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11385 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11386 attributes.set_is_evex_instruction(); 11387 attributes.set_embedded_opmask_register_specifier(mask); 11388 if (merge) { 11389 attributes.reset_is_clear_context(); 11390 } 11391 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11392 emit_int16(0x5E, (0xC0 | encode)); 11393 } 11394 11395 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11396 InstructionMark im(this); 11397 assert(VM_Version::supports_evex(), ""); 11398 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11399 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11400 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11401 attributes.set_is_evex_instruction(); 11402 attributes.set_embedded_opmask_register_specifier(mask); 11403 if (merge) { 11404 attributes.reset_is_clear_context(); 11405 } 11406 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11407 emit_int8(0x5E); 11408 emit_operand(dst, src, 0); 11409 } 11410 11411 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11412 assert(VM_Version::supports_evex(), ""); 11413 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11414 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11415 attributes.set_is_evex_instruction(); 11416 attributes.set_embedded_opmask_register_specifier(mask); 11417 if (merge) { 11418 attributes.reset_is_clear_context(); 11419 } 11420 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11421 emit_int16(0x5E, (0xC0 | encode)); 11422 } 11423 11424 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11425 InstructionMark im(this); 11426 assert(VM_Version::supports_evex(), ""); 11427 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11428 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11429 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11430 attributes.set_is_evex_instruction(); 11431 attributes.set_embedded_opmask_register_specifier(mask); 11432 if (merge) { 11433 attributes.reset_is_clear_context(); 11434 } 11435 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11436 emit_int8(0x5E); 11437 emit_operand(dst, src, 0); 11438 } 11439 11440 void Assembler::evdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src, EvexRoundPrefix rmode) { 11441 assert(VM_Version::supports_evex(), ""); 11442 InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 11443 attributes.set_extended_context(); 11444 attributes.set_is_evex_instruction(); 11445 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 11446 emit_int16(0x5E, (0xC0 | encode)); 11447 } 11448 11449 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 11450 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11451 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11452 attributes.set_is_evex_instruction(); 11453 attributes.set_embedded_opmask_register_specifier(mask); 11454 if (merge) { 11455 attributes.reset_is_clear_context(); 11456 } 11457 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11458 emit_int16(0x1C, (0xC0 | encode)); 11459 } 11460 11461 11462 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 11463 InstructionMark im(this); 11464 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11465 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11466 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11467 attributes.set_is_evex_instruction(); 11468 attributes.set_embedded_opmask_register_specifier(mask); 11469 if (merge) { 11470 attributes.reset_is_clear_context(); 11471 } 11472 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11473 emit_int8(0x1C); 11474 emit_operand(dst, src, 0); 11475 } 11476 11477 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 11478 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11479 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11480 attributes.set_is_evex_instruction(); 11481 attributes.set_embedded_opmask_register_specifier(mask); 11482 if (merge) { 11483 attributes.reset_is_clear_context(); 11484 } 11485 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11486 emit_int16(0x1D, (0xC0 | encode)); 11487 } 11488 11489 11490 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 11491 InstructionMark im(this); 11492 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11493 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11494 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit); 11495 attributes.set_is_evex_instruction(); 11496 attributes.set_embedded_opmask_register_specifier(mask); 11497 if (merge) { 11498 attributes.reset_is_clear_context(); 11499 } 11500 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11501 emit_int8(0x1D); 11502 emit_operand(dst, src, 0); 11503 } 11504 11505 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 11506 assert(VM_Version::supports_evex(), ""); 11507 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11508 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11509 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11510 attributes.set_is_evex_instruction(); 11511 attributes.set_embedded_opmask_register_specifier(mask); 11512 if (merge) { 11513 attributes.reset_is_clear_context(); 11514 } 11515 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11516 emit_int16(0x1E, (0xC0 | encode)); 11517 } 11518 11519 11520 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 11521 InstructionMark im(this); 11522 assert(VM_Version::supports_evex(), ""); 11523 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11524 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11525 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11526 attributes.set_is_evex_instruction(); 11527 attributes.set_embedded_opmask_register_specifier(mask); 11528 if (merge) { 11529 attributes.reset_is_clear_context(); 11530 } 11531 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11532 emit_int8(0x1E); 11533 emit_operand(dst, src, 0); 11534 } 11535 11536 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 11537 assert(VM_Version::supports_evex(), ""); 11538 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11539 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11540 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11541 attributes.set_is_evex_instruction(); 11542 attributes.set_embedded_opmask_register_specifier(mask); 11543 if (merge) { 11544 attributes.reset_is_clear_context(); 11545 } 11546 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11547 emit_int16(0x1F, (0xC0 | encode)); 11548 } 11549 11550 11551 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 11552 InstructionMark im(this); 11553 assert(VM_Version::supports_evex(), ""); 11554 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11555 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11556 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11557 attributes.set_is_evex_instruction(); 11558 attributes.set_embedded_opmask_register_specifier(mask); 11559 if (merge) { 11560 attributes.reset_is_clear_context(); 11561 } 11562 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11563 emit_int8(0x1F); 11564 emit_operand(dst, src, 0); 11565 } 11566 11567 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11568 assert(VM_Version::supports_evex(), ""); 11569 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11570 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11571 attributes.set_is_evex_instruction(); 11572 attributes.set_embedded_opmask_register_specifier(mask); 11573 if (merge) { 11574 attributes.reset_is_clear_context(); 11575 } 11576 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11577 emit_int16((unsigned char)0xA8, (0xC0 | encode)); 11578 } 11579 11580 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11581 InstructionMark im(this); 11582 assert(VM_Version::supports_evex(), ""); 11583 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11584 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11585 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11586 attributes.set_is_evex_instruction(); 11587 attributes.set_embedded_opmask_register_specifier(mask); 11588 if (merge) { 11589 attributes.reset_is_clear_context(); 11590 } 11591 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11592 emit_int8((unsigned char)0xA8); 11593 emit_operand(dst, src, 0); 11594 } 11595 11596 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11597 assert(VM_Version::supports_evex(), ""); 11598 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11599 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11600 attributes.set_is_evex_instruction(); 11601 attributes.set_embedded_opmask_register_specifier(mask); 11602 if (merge) { 11603 attributes.reset_is_clear_context(); 11604 } 11605 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11606 emit_int16((unsigned char)0xA8, (0xC0 | encode)); 11607 } 11608 11609 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11610 InstructionMark im(this); 11611 assert(VM_Version::supports_evex(), ""); 11612 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11613 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 11614 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit); 11615 attributes.set_is_evex_instruction(); 11616 attributes.set_embedded_opmask_register_specifier(mask); 11617 if (merge) { 11618 attributes.reset_is_clear_context(); 11619 } 11620 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11621 emit_int8((unsigned char)0xA8); 11622 emit_operand(dst, src, 0); 11623 } 11624 11625 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11626 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11627 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11628 attributes.set_is_evex_instruction(); 11629 attributes.set_embedded_opmask_register_specifier(mask); 11630 if (merge) { 11631 attributes.reset_is_clear_context(); 11632 } 11633 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11634 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 11635 } 11636 11637 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11638 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11639 InstructionMark im(this); 11640 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11641 attributes.set_is_evex_instruction(); 11642 attributes.set_embedded_opmask_register_specifier(mask); 11643 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 11644 if (merge) { 11645 attributes.reset_is_clear_context(); 11646 } 11647 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11648 emit_int8((unsigned char)0x8D); 11649 emit_operand(dst, src, 0); 11650 } 11651 11652 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11653 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11654 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11655 attributes.set_is_evex_instruction(); 11656 attributes.set_embedded_opmask_register_specifier(mask); 11657 if (merge) { 11658 attributes.reset_is_clear_context(); 11659 } 11660 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11661 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 11662 } 11663 11664 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11665 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11666 InstructionMark im(this); 11667 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11668 attributes.set_is_evex_instruction(); 11669 attributes.set_embedded_opmask_register_specifier(mask); 11670 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 11671 if (merge) { 11672 attributes.reset_is_clear_context(); 11673 } 11674 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11675 emit_int8((unsigned char)0x8D); 11676 emit_operand(dst, src, 0); 11677 } 11678 11679 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11680 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 11681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11682 attributes.set_is_evex_instruction(); 11683 attributes.set_embedded_opmask_register_specifier(mask); 11684 if (merge) { 11685 attributes.reset_is_clear_context(); 11686 } 11687 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11688 emit_int16(0x36, (0xC0 | encode)); 11689 } 11690 11691 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11692 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 11693 InstructionMark im(this); 11694 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11695 attributes.set_is_evex_instruction(); 11696 attributes.set_embedded_opmask_register_specifier(mask); 11697 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 11698 if (merge) { 11699 attributes.reset_is_clear_context(); 11700 } 11701 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11702 emit_int8(0x36); 11703 emit_operand(dst, src, 0); 11704 } 11705 11706 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11707 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 11708 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11709 attributes.set_is_evex_instruction(); 11710 attributes.set_embedded_opmask_register_specifier(mask); 11711 if (merge) { 11712 attributes.reset_is_clear_context(); 11713 } 11714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11715 emit_int16(0x36, (0xC0 | encode)); 11716 } 11717 11718 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 11719 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 11720 InstructionMark im(this); 11721 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11722 attributes.set_is_evex_instruction(); 11723 attributes.set_embedded_opmask_register_specifier(mask); 11724 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 11725 if (merge) { 11726 attributes.reset_is_clear_context(); 11727 } 11728 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11729 emit_int8(0x36); 11730 emit_operand(dst, src, 0); 11731 } 11732 11733 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11734 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11735 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11736 attributes.set_is_evex_instruction(); 11737 attributes.set_embedded_opmask_register_specifier(mask); 11738 if (merge) { 11739 attributes.reset_is_clear_context(); 11740 } 11741 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11742 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 11743 } 11744 11745 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11746 assert(VM_Version::supports_evex(), ""); 11747 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11748 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11749 attributes.set_is_evex_instruction(); 11750 attributes.set_embedded_opmask_register_specifier(mask); 11751 if (merge) { 11752 attributes.reset_is_clear_context(); 11753 } 11754 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11755 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11756 } 11757 11758 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11759 assert(VM_Version::supports_evex(), ""); 11760 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11761 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11762 attributes.set_is_evex_instruction(); 11763 attributes.set_embedded_opmask_register_specifier(mask); 11764 if (merge) { 11765 attributes.reset_is_clear_context(); 11766 } 11767 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11768 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 11769 } 11770 11771 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11772 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11773 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11774 attributes.set_is_evex_instruction(); 11775 attributes.set_embedded_opmask_register_specifier(mask); 11776 if (merge) { 11777 attributes.reset_is_clear_context(); 11778 } 11779 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11780 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 11781 } 11782 11783 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11784 assert(VM_Version::supports_evex(), ""); 11785 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11786 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11787 attributes.set_is_evex_instruction(); 11788 attributes.set_embedded_opmask_register_specifier(mask); 11789 if (merge) { 11790 attributes.reset_is_clear_context(); 11791 } 11792 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11793 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11794 } 11795 11796 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11797 assert(VM_Version::supports_evex(), ""); 11798 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11799 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11800 attributes.set_is_evex_instruction(); 11801 attributes.set_embedded_opmask_register_specifier(mask); 11802 if (merge) { 11803 attributes.reset_is_clear_context(); 11804 } 11805 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11806 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 11807 } 11808 11809 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11810 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11811 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11812 attributes.set_is_evex_instruction(); 11813 attributes.set_embedded_opmask_register_specifier(mask); 11814 if (merge) { 11815 attributes.reset_is_clear_context(); 11816 } 11817 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11818 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 11819 } 11820 11821 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11822 assert(VM_Version::supports_evex(), ""); 11823 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11824 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11825 attributes.set_is_evex_instruction(); 11826 attributes.set_embedded_opmask_register_specifier(mask); 11827 if (merge) { 11828 attributes.reset_is_clear_context(); 11829 } 11830 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11831 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11832 } 11833 11834 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11835 assert(VM_Version::supports_evex(), ""); 11836 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11837 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11838 attributes.set_is_evex_instruction(); 11839 attributes.set_embedded_opmask_register_specifier(mask); 11840 if (merge) { 11841 attributes.reset_is_clear_context(); 11842 } 11843 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11844 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11845 } 11846 11847 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11848 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11849 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11850 attributes.set_is_evex_instruction(); 11851 attributes.set_embedded_opmask_register_specifier(mask); 11852 if (merge) { 11853 attributes.reset_is_clear_context(); 11854 } 11855 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11856 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 11857 } 11858 11859 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11860 assert(VM_Version::supports_evex(), ""); 11861 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11862 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11863 attributes.set_is_evex_instruction(); 11864 attributes.set_embedded_opmask_register_specifier(mask); 11865 if (merge) { 11866 attributes.reset_is_clear_context(); 11867 } 11868 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11869 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 11870 } 11871 11872 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11873 assert(VM_Version::supports_evex(), ""); 11874 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11875 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11876 attributes.set_is_evex_instruction(); 11877 attributes.set_embedded_opmask_register_specifier(mask); 11878 if (merge) { 11879 attributes.reset_is_clear_context(); 11880 } 11881 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11882 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 11883 } 11884 11885 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11886 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11887 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11888 attributes.set_is_evex_instruction(); 11889 attributes.set_embedded_opmask_register_specifier(mask); 11890 if (merge) { 11891 attributes.reset_is_clear_context(); 11892 } 11893 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11894 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 11895 } 11896 11897 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11898 assert(VM_Version::supports_evex(), ""); 11899 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11900 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11901 attributes.set_is_evex_instruction(); 11902 attributes.set_embedded_opmask_register_specifier(mask); 11903 if (merge) { 11904 attributes.reset_is_clear_context(); 11905 } 11906 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11907 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 11908 } 11909 11910 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11911 assert(VM_Version::supports_evex(), ""); 11912 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11913 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11914 attributes.set_is_evex_instruction(); 11915 attributes.set_embedded_opmask_register_specifier(mask); 11916 if (merge) { 11917 attributes.reset_is_clear_context(); 11918 } 11919 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11920 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 11921 } 11922 11923 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11924 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11925 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11926 attributes.set_is_evex_instruction(); 11927 attributes.set_embedded_opmask_register_specifier(mask); 11928 if (merge) { 11929 attributes.reset_is_clear_context(); 11930 } 11931 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11932 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 11933 } 11934 11935 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11936 assert(VM_Version::supports_evex(), ""); 11937 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11938 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11939 attributes.set_is_evex_instruction(); 11940 attributes.set_embedded_opmask_register_specifier(mask); 11941 if (merge) { 11942 attributes.reset_is_clear_context(); 11943 } 11944 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11945 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 11946 } 11947 11948 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11949 assert(VM_Version::supports_evex(), ""); 11950 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11951 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11952 attributes.set_is_evex_instruction(); 11953 attributes.set_embedded_opmask_register_specifier(mask); 11954 if (merge) { 11955 attributes.reset_is_clear_context(); 11956 } 11957 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11958 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 11959 } 11960 11961 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11962 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 11963 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11964 attributes.set_is_evex_instruction(); 11965 attributes.set_embedded_opmask_register_specifier(mask); 11966 if (merge) { 11967 attributes.reset_is_clear_context(); 11968 } 11969 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11970 emit_int16(0x12, (0xC0 | encode)); 11971 } 11972 11973 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11974 assert(VM_Version::supports_evex(), ""); 11975 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11976 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11977 attributes.set_is_evex_instruction(); 11978 attributes.set_embedded_opmask_register_specifier(mask); 11979 if (merge) { 11980 attributes.reset_is_clear_context(); 11981 } 11982 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11983 emit_int16(0x47, (0xC0 | encode)); 11984 } 11985 11986 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11987 assert(VM_Version::supports_evex(), ""); 11988 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11989 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11990 attributes.set_is_evex_instruction(); 11991 attributes.set_embedded_opmask_register_specifier(mask); 11992 if (merge) { 11993 attributes.reset_is_clear_context(); 11994 } 11995 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11996 emit_int16(0x47, (0xC0 | encode)); 11997 } 11998 11999 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12000 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12001 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12002 attributes.set_is_evex_instruction(); 12003 attributes.set_embedded_opmask_register_specifier(mask); 12004 if (merge) { 12005 attributes.reset_is_clear_context(); 12006 } 12007 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12008 emit_int16(0x10, (0xC0 | encode)); 12009 } 12010 12011 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12012 assert(VM_Version::supports_evex(), ""); 12013 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12014 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12015 attributes.set_is_evex_instruction(); 12016 attributes.set_embedded_opmask_register_specifier(mask); 12017 if (merge) { 12018 attributes.reset_is_clear_context(); 12019 } 12020 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12021 emit_int16(0x45, (0xC0 | encode)); 12022 } 12023 12024 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12025 assert(VM_Version::supports_evex(), ""); 12026 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12027 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12028 attributes.set_is_evex_instruction(); 12029 attributes.set_embedded_opmask_register_specifier(mask); 12030 if (merge) { 12031 attributes.reset_is_clear_context(); 12032 } 12033 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12034 emit_int16(0x45, (0xC0 | encode)); 12035 } 12036 12037 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12038 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12039 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12040 attributes.set_is_evex_instruction(); 12041 attributes.set_embedded_opmask_register_specifier(mask); 12042 if (merge) { 12043 attributes.reset_is_clear_context(); 12044 } 12045 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12046 emit_int16(0x11, (0xC0 | encode)); 12047 } 12048 12049 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12050 assert(VM_Version::supports_evex(), ""); 12051 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12052 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12053 attributes.set_is_evex_instruction(); 12054 attributes.set_embedded_opmask_register_specifier(mask); 12055 if (merge) { 12056 attributes.reset_is_clear_context(); 12057 } 12058 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12059 emit_int16(0x46, (0xC0 | encode)); 12060 } 12061 12062 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12063 assert(VM_Version::supports_evex(), ""); 12064 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12065 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12066 attributes.set_is_evex_instruction(); 12067 attributes.set_embedded_opmask_register_specifier(mask); 12068 if (merge) { 12069 attributes.reset_is_clear_context(); 12070 } 12071 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12072 emit_int16(0x46, (0xC0 | encode)); 12073 } 12074 12075 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12076 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12077 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12078 attributes.set_is_evex_instruction(); 12079 attributes.set_embedded_opmask_register_specifier(mask); 12080 if (merge) { 12081 attributes.reset_is_clear_context(); 12082 } 12083 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12084 emit_int16(0x38, (0xC0 | encode)); 12085 } 12086 12087 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12088 assert(VM_Version::supports_avx512bw(), ""); 12089 InstructionMark im(this); 12090 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12091 attributes.set_is_evex_instruction(); 12092 attributes.set_embedded_opmask_register_specifier(mask); 12093 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 12094 if (merge) { 12095 attributes.reset_is_clear_context(); 12096 } 12097 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12098 emit_int8(0x38); 12099 emit_operand(dst, src, 0); 12100 } 12101 12102 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12103 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12104 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12105 attributes.set_is_evex_instruction(); 12106 attributes.set_embedded_opmask_register_specifier(mask); 12107 if (merge) { 12108 attributes.reset_is_clear_context(); 12109 } 12110 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 12111 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 12112 } 12113 12114 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12115 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12116 InstructionMark im(this); 12117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12118 attributes.set_is_evex_instruction(); 12119 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 12120 attributes.set_embedded_opmask_register_specifier(mask); 12121 if (merge) { 12122 attributes.reset_is_clear_context(); 12123 } 12124 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 12125 emit_int8((unsigned char)0xEA); 12126 emit_operand(dst, src, 0); 12127 } 12128 12129 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12130 assert(VM_Version::supports_evex(), ""); 12131 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12132 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12133 attributes.set_is_evex_instruction(); 12134 attributes.set_embedded_opmask_register_specifier(mask); 12135 if (merge) { 12136 attributes.reset_is_clear_context(); 12137 } 12138 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12139 emit_int16(0x39, (0xC0 | encode)); 12140 } 12141 12142 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12143 assert(VM_Version::supports_evex(), ""); 12144 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12145 InstructionMark im(this); 12146 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12147 attributes.set_is_evex_instruction(); 12148 attributes.set_embedded_opmask_register_specifier(mask); 12149 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12150 if (merge) { 12151 attributes.reset_is_clear_context(); 12152 } 12153 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12154 emit_int8(0x39); 12155 emit_operand(dst, src, 0); 12156 } 12157 12158 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12159 assert(VM_Version::supports_evex(), ""); 12160 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12161 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12162 attributes.set_is_evex_instruction(); 12163 attributes.set_embedded_opmask_register_specifier(mask); 12164 if (merge) { 12165 attributes.reset_is_clear_context(); 12166 } 12167 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12168 emit_int16(0x39, (0xC0 | encode)); 12169 } 12170 12171 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12172 assert(VM_Version::supports_evex(), ""); 12173 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12174 InstructionMark im(this); 12175 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12176 attributes.set_is_evex_instruction(); 12177 attributes.set_embedded_opmask_register_specifier(mask); 12178 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12179 if (merge) { 12180 attributes.reset_is_clear_context(); 12181 } 12182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12183 emit_int8(0x39); 12184 emit_operand(dst, src, 0); 12185 } 12186 12187 12188 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12189 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12190 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12191 attributes.set_is_evex_instruction(); 12192 attributes.set_embedded_opmask_register_specifier(mask); 12193 if (merge) { 12194 attributes.reset_is_clear_context(); 12195 } 12196 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12197 emit_int16(0x3C, (0xC0 | encode)); 12198 } 12199 12200 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12201 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12202 InstructionMark im(this); 12203 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12204 attributes.set_is_evex_instruction(); 12205 attributes.set_embedded_opmask_register_specifier(mask); 12206 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 12207 if (merge) { 12208 attributes.reset_is_clear_context(); 12209 } 12210 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12211 emit_int8(0x3C); 12212 emit_operand(dst, src, 0); 12213 } 12214 12215 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12216 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12217 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12218 attributes.set_is_evex_instruction(); 12219 attributes.set_embedded_opmask_register_specifier(mask); 12220 if (merge) { 12221 attributes.reset_is_clear_context(); 12222 } 12223 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 12224 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 12225 } 12226 12227 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12228 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 12229 InstructionMark im(this); 12230 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12231 attributes.set_is_evex_instruction(); 12232 attributes.set_embedded_opmask_register_specifier(mask); 12233 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 12234 if (merge) { 12235 attributes.reset_is_clear_context(); 12236 } 12237 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 12238 emit_int8((unsigned char)0xEE); 12239 emit_operand(dst, src, 0); 12240 } 12241 12242 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12243 assert(VM_Version::supports_evex(), ""); 12244 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12245 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12246 attributes.set_is_evex_instruction(); 12247 attributes.set_embedded_opmask_register_specifier(mask); 12248 if (merge) { 12249 attributes.reset_is_clear_context(); 12250 } 12251 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12252 emit_int16(0x3D, (0xC0 | encode)); 12253 } 12254 12255 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12256 assert(VM_Version::supports_evex(), ""); 12257 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12258 InstructionMark im(this); 12259 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12260 attributes.set_is_evex_instruction(); 12261 attributes.set_embedded_opmask_register_specifier(mask); 12262 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12263 if (merge) { 12264 attributes.reset_is_clear_context(); 12265 } 12266 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12267 emit_int8(0x3D); 12268 emit_operand(dst, src, 0); 12269 } 12270 12271 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12272 assert(VM_Version::supports_evex(), ""); 12273 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12274 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12275 attributes.set_is_evex_instruction(); 12276 attributes.set_embedded_opmask_register_specifier(mask); 12277 if (merge) { 12278 attributes.reset_is_clear_context(); 12279 } 12280 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12281 emit_int16(0x3D, (0xC0 | encode)); 12282 } 12283 12284 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 12285 assert(VM_Version::supports_evex(), ""); 12286 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12287 InstructionMark im(this); 12288 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12289 attributes.set_is_evex_instruction(); 12290 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12291 attributes.set_embedded_opmask_register_specifier(mask); 12292 if (merge) { 12293 attributes.reset_is_clear_context(); 12294 } 12295 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12296 emit_int8(0x3D); 12297 emit_operand(dst, src, 0); 12298 } 12299 12300 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) { 12301 assert(VM_Version::supports_evex(), "requires EVEX support"); 12302 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 12303 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12304 attributes.set_is_evex_instruction(); 12305 attributes.set_embedded_opmask_register_specifier(mask); 12306 if (merge) { 12307 attributes.reset_is_clear_context(); 12308 } 12309 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12310 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8); 12311 } 12312 12313 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) { 12314 assert(VM_Version::supports_evex(), "requires EVEX support"); 12315 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 12316 assert(dst != xnoreg, "sanity"); 12317 InstructionMark im(this); 12318 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12319 attributes.set_is_evex_instruction(); 12320 attributes.set_embedded_opmask_register_specifier(mask); 12321 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12322 if (merge) { 12323 attributes.reset_is_clear_context(); 12324 } 12325 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12326 emit_int8(0x25); 12327 emit_operand(dst, src3, 1); 12328 emit_int8(imm8); 12329 } 12330 12331 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) { 12332 assert(VM_Version::supports_evex(), "requires EVEX support"); 12333 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 12334 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12335 attributes.set_is_evex_instruction(); 12336 attributes.set_embedded_opmask_register_specifier(mask); 12337 if (merge) { 12338 attributes.reset_is_clear_context(); 12339 } 12340 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12341 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8); 12342 } 12343 12344 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) { 12345 assert(VM_Version::supports_evex(), "requires EVEX support"); 12346 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 12347 assert(dst != xnoreg, "sanity"); 12348 InstructionMark im(this); 12349 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12350 attributes.set_is_evex_instruction(); 12351 attributes.set_embedded_opmask_register_specifier(mask); 12352 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 12353 if (merge) { 12354 attributes.reset_is_clear_context(); 12355 } 12356 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12357 emit_int8(0x25); 12358 emit_operand(dst, src3, 1); 12359 emit_int8(imm8); 12360 } 12361 12362 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) { 12363 assert(VM_Version::supports_gfni(), ""); 12364 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 12365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12366 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8); 12367 } 12368 12369 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) { 12370 assert(VM_Version::supports_gfni(), "requires GFNI support"); 12371 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12372 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12373 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8); 12374 } 12375 12376 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL 12377 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) { 12378 assert(UseAVX >= 2, ""); 12379 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12380 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12381 emit_int16(0x58, (0xC0 | encode)); 12382 } 12383 12384 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) { 12385 assert(VM_Version::supports_avx2(), ""); 12386 assert(dst != xnoreg, "sanity"); 12387 InstructionMark im(this); 12388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12389 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12390 // swap src<->dst for encoding 12391 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12392 emit_int8(0x58); 12393 emit_operand(dst, src, 0); 12394 } 12395 12396 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL 12397 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) { 12398 assert(VM_Version::supports_avx2(), ""); 12399 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12400 attributes.set_rex_vex_w_reverted(); 12401 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12402 emit_int16(0x59, (0xC0 | encode)); 12403 } 12404 12405 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) { 12406 assert(VM_Version::supports_avx2(), ""); 12407 assert(dst != xnoreg, "sanity"); 12408 InstructionMark im(this); 12409 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12410 attributes.set_rex_vex_w_reverted(); 12411 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12412 // swap src<->dst for encoding 12413 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12414 emit_int8(0x59); 12415 emit_operand(dst, src, 0); 12416 } 12417 12418 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) { 12419 assert(vector_len != Assembler::AVX_128bit, ""); 12420 assert(VM_Version::supports_evex(), ""); 12421 assert(dst != xnoreg, "sanity"); 12422 InstructionMark im(this); 12423 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12424 attributes.set_rex_vex_w_reverted(); 12425 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 12426 // swap src<->dst for encoding 12427 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12428 emit_int8(0x5A); 12429 emit_operand(dst, src, 0); 12430 } 12431 12432 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) { 12433 assert(vector_len != Assembler::AVX_128bit, ""); 12434 assert(VM_Version::supports_avx512dq(), ""); 12435 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12436 attributes.set_rex_vex_w_reverted(); 12437 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12438 emit_int16(0x5A, (0xC0 | encode)); 12439 } 12440 12441 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) { 12442 assert(vector_len != Assembler::AVX_128bit, ""); 12443 assert(VM_Version::supports_avx512dq(), ""); 12444 assert(dst != xnoreg, "sanity"); 12445 InstructionMark im(this); 12446 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12447 attributes.set_rex_vex_w_reverted(); 12448 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit); 12449 // swap src<->dst for encoding 12450 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12451 emit_int8(0x5A); 12452 emit_operand(dst, src, 0); 12453 } 12454 12455 void Assembler::vbroadcasti128(XMMRegister dst, Address src, int vector_len) { 12456 assert(VM_Version::supports_avx2(), ""); 12457 assert(vector_len == AVX_256bit, ""); 12458 assert(dst != xnoreg, "sanity"); 12459 InstructionMark im(this); 12460 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12461 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 12462 // swap src<->dst for encoding 12463 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12464 emit_int8(0x5A); 12465 emit_operand(dst, src, 0); 12466 } 12467 12468 // scalar single/double precision replicate 12469 12470 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL 12471 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) { 12472 assert(VM_Version::supports_avx2(), ""); 12473 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12474 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12475 emit_int16(0x18, (0xC0 | encode)); 12476 } 12477 12478 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) { 12479 assert(VM_Version::supports_avx(), ""); 12480 assert(dst != xnoreg, "sanity"); 12481 InstructionMark im(this); 12482 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12483 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12484 // swap src<->dst for encoding 12485 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12486 emit_int8(0x18); 12487 emit_operand(dst, src, 0); 12488 } 12489 12490 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL 12491 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) { 12492 assert(VM_Version::supports_avx2(), ""); 12493 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, ""); 12494 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12495 attributes.set_rex_vex_w_reverted(); 12496 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12497 emit_int16(0x19, (0xC0 | encode)); 12498 } 12499 12500 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) { 12501 assert(VM_Version::supports_avx(), ""); 12502 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, ""); 12503 assert(dst != xnoreg, "sanity"); 12504 InstructionMark im(this); 12505 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12506 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12507 attributes.set_rex_vex_w_reverted(); 12508 // swap src<->dst for encoding 12509 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12510 emit_int8(0x19); 12511 emit_operand(dst, src, 0); 12512 } 12513 12514 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) { 12515 assert(VM_Version::supports_avx(), ""); 12516 assert(vector_len == AVX_256bit, ""); 12517 assert(dst != xnoreg, "sanity"); 12518 InstructionMark im(this); 12519 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12520 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 12521 // swap src<->dst for encoding 12522 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12523 emit_int8(0x1A); 12524 emit_operand(dst, src, 0); 12525 } 12526 12527 void Assembler::evbroadcastf64x2(XMMRegister dst, Address src, int vector_len) { 12528 assert(VM_Version::supports_avx512dq(), ""); 12529 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12530 assert(dst != xnoreg, "sanity"); 12531 InstructionMark im(this); 12532 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12533 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit); 12534 attributes.set_is_evex_instruction(); 12535 // swap src<->dst for encoding 12536 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12537 emit_int8(0x1A); 12538 emit_operand(dst, src, 0); 12539 } 12540 12541 12542 // gpr source broadcast forms 12543 12544 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 12545 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) { 12546 assert(VM_Version::supports_avx512bw(), ""); 12547 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 12548 attributes.set_is_evex_instruction(); 12549 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true); 12550 emit_int16(0x7A, (0xC0 | encode)); 12551 } 12552 12553 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 12554 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) { 12555 assert(VM_Version::supports_avx512bw(), ""); 12556 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 12557 attributes.set_is_evex_instruction(); 12558 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes , true); 12559 emit_int16(0x7B, (0xC0 | encode)); 12560 } 12561 12562 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL 12563 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) { 12564 assert(VM_Version::supports_evex(), ""); 12565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12566 attributes.set_is_evex_instruction(); 12567 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true); 12568 emit_int16(0x7C, (0xC0 | encode)); 12569 } 12570 12571 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL 12572 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) { 12573 assert(VM_Version::supports_evex(), ""); 12574 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12575 attributes.set_is_evex_instruction(); 12576 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true); 12577 emit_int16(0x7C, (0xC0 | encode)); 12578 } 12579 12580 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 12581 assert(VM_Version::supports_avx2(), ""); 12582 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand"); 12583 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 12584 assert(dst != xnoreg, "sanity"); 12585 assert(src.isxmmindex(),"expected to be xmm index"); 12586 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12587 InstructionMark im(this); 12588 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12589 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12590 emit_int8((unsigned char)0x90); 12591 emit_operand(dst, src, 0); 12592 } 12593 12594 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 12595 assert(VM_Version::supports_avx2(), ""); 12596 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand"); 12597 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 12598 assert(dst != xnoreg, "sanity"); 12599 assert(src.isxmmindex(),"expected to be xmm index"); 12600 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12601 InstructionMark im(this); 12602 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12603 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12604 emit_int8((unsigned char)0x90); 12605 emit_operand(dst, src, 0); 12606 } 12607 12608 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 12609 assert(VM_Version::supports_avx2(), ""); 12610 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand"); 12611 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 12612 assert(dst != xnoreg, "sanity"); 12613 assert(src.isxmmindex(),"expected to be xmm index"); 12614 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12615 InstructionMark im(this); 12616 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12617 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12618 emit_int8((unsigned char)0x92); 12619 emit_operand(dst, src, 0); 12620 } 12621 12622 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 12623 assert(VM_Version::supports_avx2(), ""); 12624 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand"); 12625 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 12626 assert(dst != xnoreg, "sanity"); 12627 assert(src.isxmmindex(),"expected to be xmm index"); 12628 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12629 InstructionMark im(this); 12630 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false); 12631 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12632 emit_int8((unsigned char)0x92); 12633 emit_operand(dst, src, 0); 12634 } 12635 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 12636 assert(VM_Version::supports_evex(), ""); 12637 assert(dst != xnoreg, "sanity"); 12638 assert(src.isxmmindex(),"expected to be xmm index"); 12639 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12640 assert(mask != k0, "instruction will #UD if mask is in k0"); 12641 InstructionMark im(this); 12642 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12643 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12644 attributes.reset_is_clear_context(); 12645 attributes.set_embedded_opmask_register_specifier(mask); 12646 attributes.set_is_evex_instruction(); 12647 // swap src<->dst for encoding 12648 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12649 emit_int8((unsigned char)0x90); 12650 emit_operand(dst, src, 0); 12651 } 12652 12653 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) { 12654 assert(VM_Version::supports_evex(), ""); 12655 assert(dst != xnoreg, "sanity"); 12656 assert(src.isxmmindex(),"expected to be xmm index"); 12657 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12658 assert(mask != k0, "instruction will #UD if mask is in k0"); 12659 InstructionMark im(this); 12660 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12661 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12662 attributes.reset_is_clear_context(); 12663 attributes.set_embedded_opmask_register_specifier(mask); 12664 attributes.set_is_evex_instruction(); 12665 // swap src<->dst for encoding 12666 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12667 emit_int8((unsigned char)0x90); 12668 emit_operand(dst, src, 0); 12669 } 12670 12671 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 12672 assert(VM_Version::supports_evex(), ""); 12673 assert(dst != xnoreg, "sanity"); 12674 assert(src.isxmmindex(),"expected to be xmm index"); 12675 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12676 assert(mask != k0, "instruction will #UD if mask is in k0"); 12677 InstructionMark im(this); 12678 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12679 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12680 attributes.reset_is_clear_context(); 12681 attributes.set_embedded_opmask_register_specifier(mask); 12682 attributes.set_is_evex_instruction(); 12683 // swap src<->dst for encoding 12684 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12685 emit_int8((unsigned char)0x92); 12686 emit_operand(dst, src, 0); 12687 } 12688 12689 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) { 12690 assert(VM_Version::supports_evex(), ""); 12691 assert(dst != xnoreg, "sanity"); 12692 assert(src.isxmmindex(),"expected to be xmm index"); 12693 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 12694 assert(mask != k0, "instruction will #UD if mask is in k0"); 12695 InstructionMark im(this); 12696 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12697 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12698 attributes.reset_is_clear_context(); 12699 attributes.set_embedded_opmask_register_specifier(mask); 12700 attributes.set_is_evex_instruction(); 12701 // swap src<->dst for encoding 12702 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12703 emit_int8((unsigned char)0x92); 12704 emit_operand(dst, src, 0); 12705 } 12706 12707 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) { 12708 assert(VM_Version::supports_evex(), ""); 12709 assert(mask != k0, "instruction will #UD if mask is in k0"); 12710 InstructionMark im(this); 12711 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12712 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12713 attributes.reset_is_clear_context(); 12714 attributes.set_embedded_opmask_register_specifier(mask); 12715 attributes.set_is_evex_instruction(); 12716 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12717 emit_int8((unsigned char)0xA0); 12718 emit_operand(src, dst, 0); 12719 } 12720 12721 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) { 12722 assert(VM_Version::supports_evex(), ""); 12723 assert(mask != k0, "instruction will #UD if mask is in k0"); 12724 InstructionMark im(this); 12725 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12726 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12727 attributes.reset_is_clear_context(); 12728 attributes.set_embedded_opmask_register_specifier(mask); 12729 attributes.set_is_evex_instruction(); 12730 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12731 emit_int8((unsigned char)0xA0); 12732 emit_operand(src, dst, 0); 12733 } 12734 12735 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) { 12736 assert(VM_Version::supports_evex(), ""); 12737 assert(mask != k0, "instruction will #UD if mask is in k0"); 12738 InstructionMark im(this); 12739 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12740 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 12741 attributes.reset_is_clear_context(); 12742 attributes.set_embedded_opmask_register_specifier(mask); 12743 attributes.set_is_evex_instruction(); 12744 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12745 emit_int8((unsigned char)0xA2); 12746 emit_operand(src, dst, 0); 12747 } 12748 12749 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) { 12750 assert(VM_Version::supports_evex(), ""); 12751 assert(mask != k0, "instruction will #UD if mask is in k0"); 12752 InstructionMark im(this); 12753 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12754 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 12755 attributes.reset_is_clear_context(); 12756 attributes.set_embedded_opmask_register_specifier(mask); 12757 attributes.set_is_evex_instruction(); 12758 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12759 emit_int8((unsigned char)0xA2); 12760 emit_operand(src, dst, 0); 12761 } 12762 // Carry-Less Multiplication Quadword 12763 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) { 12764 assert(VM_Version::supports_clmul(), ""); 12765 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12766 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12767 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 12768 } 12769 12770 // Carry-Less Multiplication Quadword 12771 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) { 12772 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), ""); 12773 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12774 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12775 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 12776 } 12777 12778 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) { 12779 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support"); 12780 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12781 attributes.set_is_evex_instruction(); 12782 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12783 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 12784 } 12785 12786 void Assembler::vzeroupper_uncached() { 12787 if (VM_Version::supports_vzeroupper()) { 12788 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12789 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 12790 emit_int8(0x77); 12791 } 12792 } 12793 12794 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) { 12795 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib 12796 assert(VM_Version::supports_evex(), ""); 12797 assert(VM_Version::supports_avx512dq(), ""); 12798 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false); 12799 attributes.set_is_evex_instruction(); 12800 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12801 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8); 12802 } 12803 12804 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) { 12805 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib 12806 assert(VM_Version::supports_evex(), ""); 12807 assert(VM_Version::supports_avx512dq(), ""); 12808 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false); 12809 attributes.set_is_evex_instruction(); 12810 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 12811 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8); 12812 } 12813 12814 void Assembler::fld_x(Address adr) { 12815 InstructionMark im(this); 12816 emit_int8((unsigned char)0xDB); 12817 emit_operand32(rbp, adr, 0); 12818 } 12819 12820 void Assembler::fstp_x(Address adr) { 12821 InstructionMark im(this); 12822 emit_int8((unsigned char)0xDB); 12823 emit_operand32(rdi, adr, 0); 12824 } 12825 12826 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) { 12827 assert(reg->encoding() < 8, "no extended registers"); 12828 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers"); 12829 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 12830 } 12831 12832 void Assembler::fld_d(Address adr) { 12833 InstructionMark im(this); 12834 emit_int8((unsigned char)0xDD); 12835 emit_operand32(rax, adr, 0); 12836 } 12837 12838 void Assembler::fprem() { 12839 emit_int16((unsigned char)0xD9, (unsigned char)0xF8); 12840 } 12841 12842 void Assembler::fnstsw_ax() { 12843 emit_int16((unsigned char)0xDF, (unsigned char)0xE0); 12844 } 12845 12846 void Assembler::fstp_d(Address adr) { 12847 InstructionMark im(this); 12848 emit_int8((unsigned char)0xDD); 12849 emit_operand32(rbx, adr, 0); 12850 } 12851 12852 void Assembler::fstp_d(int index) { 12853 emit_farith(0xDD, 0xD8, index); 12854 } 12855 12856 void Assembler::emit_farith(int b1, int b2, int i) { 12857 assert(isByte(b1) && isByte(b2), "wrong opcode"); 12858 assert(0 <= i && i < 8, "illegal stack offset"); 12859 emit_int16(b1, b2 + i); 12860 } 12861 12862 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding. 12863 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 }; 12864 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding. 12865 static int simd_opc[4] = { 0, 0, 0x38, 0x3A }; 12866 12867 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding. 12868 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) { 12869 if (pre > 0) { 12870 emit_int8(simd_pre[pre]); 12871 } 12872 if (rex_w) { 12873 prefixq(adr, xreg); 12874 } else { 12875 prefix(adr, xreg); 12876 } 12877 if (opc > 0) { 12878 emit_int8(0x0F); 12879 int opc2 = simd_opc[opc]; 12880 if (opc2 > 0) { 12881 emit_int8(opc2); 12882 } 12883 } 12884 } 12885 12886 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) { 12887 if (pre > 0) { 12888 emit_int8(simd_pre[pre]); 12889 } 12890 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc); 12891 if (opc > 0) { 12892 emit_int8(0x0F); 12893 int opc2 = simd_opc[opc]; 12894 if (opc2 > 0) { 12895 emit_int8(opc2); 12896 } 12897 } 12898 return encode; 12899 } 12900 12901 12902 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) { 12903 int vector_len = _attributes->get_vector_len(); 12904 bool vex_w = _attributes->is_rex_vex_w(); 12905 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) { 12906 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0); 12907 byte1 = (~byte1) & 0xE0; 12908 byte1 |= opc; 12909 12910 int byte2 = ((~nds_enc) & 0xf) << 3; 12911 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre; 12912 12913 emit_int24((unsigned char)VEX_3bytes, byte1, byte2); 12914 } else { 12915 int byte1 = vex_r ? VEX_R : 0; 12916 byte1 = (~byte1) & 0x80; 12917 byte1 |= ((~nds_enc) & 0xf) << 3; 12918 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre; 12919 emit_int16((unsigned char)VEX_2bytes, byte1); 12920 } 12921 } 12922 12923 // This is a 4 byte encoding 12924 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool eevex_b, bool evex_v, 12925 bool eevex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool no_flags) { 12926 // EVEX 0x62 prefix 12927 // byte1 = EVEX_4bytes; 12928 12929 bool vex_w = _attributes->is_rex_vex_w(); 12930 int evex_encoding = (vex_w ? VEX_W : 0); 12931 // EVEX.b is not currently used for broadcast of single element or data rounding modes 12932 _attributes->set_evex_encoding(evex_encoding); 12933 12934 // P0: byte 2, initialized to RXBR'0mmm 12935 // instead of not'd 12936 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0); 12937 byte2 = (~byte2) & 0xF0; 12938 byte2 |= eevex_b ? EEVEX_B : 0; 12939 // confine opc opcode extensions in mm bits to lower two bits 12940 // of form {0F, 0F_38, 0F_3A, 0F_3C} 12941 byte2 |= opc; 12942 12943 // P1: byte 3 as Wvvvv1pp 12944 int byte3 = ((~nds_enc) & 0xf) << 3; 12945 byte3 |= (eevex_x ? 0 : EEVEX_X); 12946 byte3 |= (vex_w & 1) << 7; 12947 // confine pre opcode extensions in pp bits to lower two bits 12948 // of form {66, F3, F2} 12949 byte3 |= pre; 12950 12951 // P2: byte 4 as zL'Lbv'aaa or 00LXVF00 where V = V4, X(extended context) = ND and F = NF (no flags) 12952 int byte4 = 0; 12953 if (no_flags) { 12954 assert(_attributes->is_no_reg_mask(), "mask register not supported with no_flags"); 12955 byte4 |= 0x4; 12956 } else { 12957 // kregs are implemented in the low 3 bits as aaa 12958 byte4 = (_attributes->is_no_reg_mask()) ? 12959 0 : 12960 _attributes->get_embedded_opmask_register_specifier(); 12961 } 12962 // EVEX.v` for extending EVEX.vvvv or VIDX 12963 byte4 |= (evex_v ? 0: EVEX_V); 12964 // third EXEC.b for broadcast actions 12965 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0); 12966 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024 12967 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5; 12968 // last is EVEX.z for zero/merge actions 12969 if (_attributes->is_no_reg_mask() == false && 12970 _attributes->get_embedded_opmask_register_specifier() != 0) { 12971 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0); 12972 } 12973 emit_int32(EVEX_4bytes, byte2, byte3, byte4); 12974 } 12975 12976 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool nds_is_ndd, bool no_flags) { 12977 if (adr.base_needs_rex2() || adr.index_needs_rex2() || nds_is_ndd || no_flags) { 12978 assert(UseAPX, "APX features not enabled"); 12979 } 12980 if (nds_is_ndd) attributes->set_extended_context(); 12981 bool is_extended = adr.base_needs_rex2() || adr.index_needs_rex2() || nds_enc >= 16 || xreg_enc >= 16 || nds_is_ndd; 12982 bool vex_r = (xreg_enc & 8) == 8; 12983 bool vex_b = adr.base_needs_rex(); 12984 bool vex_x; 12985 if (adr.isxmmindex()) { 12986 vex_x = adr.xmmindex_needs_rex(); 12987 } else { 12988 vex_x = adr.index_needs_rex(); 12989 } 12990 set_attributes(attributes); 12991 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction 12992 // is allowed in legacy mode and has resources which will fit in it. 12993 // Pure EVEX instructions will have is_evex_instruction set in their definition. 12994 if (!attributes->is_legacy_mode()) { 12995 if (UseAVX > 2 && !attributes->is_evex_instruction()) { 12996 if ((attributes->get_vector_len() != AVX_512bit) && !is_extended) { 12997 attributes->set_is_legacy_mode(); 12998 } 12999 } 13000 } 13001 13002 if (UseAVX > 2) { 13003 assert(((!attributes->uses_vl()) || 13004 (attributes->get_vector_len() == AVX_512bit) || 13005 (!_legacy_mode_vl) || 13006 (attributes->is_legacy_mode())),"XMM register should be 0-15"); 13007 assert((!is_extended || (!attributes->is_legacy_mode())),"XMM register should be 0-15"); 13008 } 13009 13010 if (UseAVX > 2 && !attributes->is_legacy_mode()) 13011 { 13012 bool evex_r = (xreg_enc >= 16); 13013 bool evex_v; 13014 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31) 13015 if (adr.isxmmindex()) { 13016 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false); 13017 } else { 13018 evex_v = (nds_enc >= 16); 13019 } 13020 bool eevex_x = adr.index_needs_rex2(); 13021 bool eevex_b = adr.base_needs_rex2(); 13022 attributes->set_is_evex_instruction(); 13023 evex_prefix(vex_r, vex_b, vex_x, evex_r, eevex_b, evex_v, eevex_x, nds_enc, pre, opc, no_flags); 13024 } else { 13025 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) { 13026 attributes->set_rex_vex_w(false); 13027 } 13028 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc); 13029 } 13030 } 13031 13032 void Assembler::eevex_prefix_ndd(Address adr, int ndd_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) { 13033 attributes->set_is_evex_instruction(); 13034 vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ true, no_flags); 13035 } 13036 13037 void Assembler::emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc, 13038 int size, int opcode_byte, bool no_flags, bool is_map1, bool is_commutative) { 13039 if (is_commutative && is_demotable(no_flags, dst->encoding(), src2->encoding())) { 13040 // Opcode byte adjustment due to mismatch between NDD and equivalent demotable variant 13041 opcode_byte += 2; 13042 if (size == EVEX_64bit) { 13043 emit_prefix_and_int8(get_prefixq(src1, dst, is_map1), opcode_byte); 13044 } else { 13045 // For 32-bit, 16-bit and 8-bit 13046 if (size == EVEX_16bit) { 13047 emit_int8(0x66); 13048 } 13049 prefix(src1, dst, false, is_map1); 13050 emit_int8(opcode_byte); 13051 } 13052 } else { 13053 bool vex_w = (size == EVEX_64bit) ? true : false; 13054 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13055 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size); 13056 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags); 13057 emit_int8(opcode_byte); 13058 } 13059 emit_operand(src2, src1, 0); 13060 } 13061 13062 void Assembler::emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc, 13063 int size, int opcode_byte, bool no_flags, bool is_map1) { 13064 if (is_demotable(no_flags, dst->encoding(), src1->encoding())) { 13065 if (size == EVEX_64bit) { 13066 emit_prefix_and_int8(get_prefixq(src2, dst, is_map1), opcode_byte); 13067 } else { 13068 // For 32-bit, 16-bit and 8-bit 13069 if (size == EVEX_16bit) { 13070 emit_int8(0x66); 13071 } 13072 prefix(src2, dst, false, is_map1); 13073 emit_int8(opcode_byte); 13074 } 13075 } else { 13076 bool vex_w = (size == EVEX_64bit) ? true : false; 13077 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13078 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size); 13079 eevex_prefix_ndd(src2, dst->encoding(), src1->encoding(), pre, opc, &attributes, no_flags); 13080 emit_int8(opcode_byte); 13081 } 13082 emit_operand(src1, src2, 0); 13083 } 13084 13085 void Assembler::eevex_prefix_nf(Address adr, int ndd_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) { 13086 attributes->set_is_evex_instruction(); 13087 vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ false, no_flags); 13088 } 13089 13090 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool src_is_gpr, bool nds_is_ndd, bool no_flags) { 13091 if (nds_is_ndd || no_flags || (src_is_gpr && src_enc >= 16)) { 13092 assert(UseAPX, "APX features not enabled"); 13093 } 13094 if (nds_is_ndd) attributes->set_extended_context(); 13095 bool is_extended = dst_enc >= 16 || nds_enc >= 16 || src_enc >=16; 13096 bool vex_r = (dst_enc & 8) == 8; 13097 bool vex_b = (src_enc & 8) == 8; 13098 bool vex_x = false; 13099 set_attributes(attributes); 13100 13101 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction 13102 // is allowed in legacy mode and has resources which will fit in it. 13103 // Pure EVEX instructions will have is_evex_instruction set in their definition. 13104 if (!attributes->is_legacy_mode()) { 13105 if (UseAVX > 2 && !attributes->is_evex_instruction()) { 13106 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) && 13107 !is_extended) { 13108 attributes->set_is_legacy_mode(); 13109 } 13110 } 13111 } 13112 13113 if (UseAVX > 2) { 13114 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false 13115 // Instruction with uses_vl true are vector instructions 13116 // All the vector instructions with AVX_512bit length can have legacy_mode as false 13117 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported 13118 // Rest all should have legacy_mode set as true 13119 assert(((!attributes->uses_vl()) || 13120 (attributes->get_vector_len() == AVX_512bit) || 13121 (!_legacy_mode_vl) || 13122 (attributes->is_legacy_mode())),"XMM register should be 0-15"); 13123 // Instruction with legacy_mode true should have dst, nds and src < 15 13124 assert(((!is_extended) || (!attributes->is_legacy_mode())),"XMM register should be 0-15"); 13125 } 13126 13127 if (UseAVX > 2 && !attributes->is_legacy_mode()) 13128 { 13129 bool evex_r = (dst_enc >= 16); 13130 bool evex_v = (nds_enc >= 16); 13131 bool evex_b = (src_enc >= 16) && src_is_gpr; 13132 // can use vex_x as bank extender on rm encoding 13133 vex_x = (src_enc >= 16) && !src_is_gpr; 13134 attributes->set_is_evex_instruction(); 13135 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_b, evex_v, false /*eevex_x*/, nds_enc, pre, opc, no_flags); 13136 } else { 13137 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) { 13138 attributes->set_rex_vex_w(false); 13139 } 13140 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc); 13141 } 13142 13143 // return modrm byte components for operands 13144 return (((dst_enc & 7) << 3) | (src_enc & 7)); 13145 } 13146 13147 void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8_t imm8, VexSimdPrefix pre, VexOpcode opc, 13148 int size, int opcode_byte, bool no_flags, bool is_map1) { 13149 bool is_prefixq = (size == EVEX_64bit) ? true : false; 13150 if (is_demotable(no_flags, dst_enc, nds_enc)) { 13151 int encode = is_prefixq ? prefixq_and_encode(src_enc, dst_enc, is_map1) : prefix_and_encode(src_enc, dst_enc, is_map1); 13152 emit_opcode_prefix_and_encoding((unsigned char)(opcode_byte | 0x80), 0xC0, encode, imm8); 13153 } else { 13154 InstructionAttr attributes(AVX_128bit, is_prefixq, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13155 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size); 13156 int encode = emit_eevex_prefix_or_demote_ndd(src_enc, dst_enc, nds_enc, pre, opc, &attributes, no_flags); 13157 emit_int24(opcode_byte, (0xC0 | encode), imm8); 13158 } 13159 } 13160 13161 void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, 13162 int size, int opcode_byte, bool no_flags, bool is_map1, bool swap, bool is_commutative) { 13163 int encode; 13164 bool is_prefixq = (size == EVEX_64bit) ? true : false; 13165 bool first_operand_demotable = is_demotable(no_flags, dst_enc, nds_enc); 13166 bool second_operand_demotable = is_commutative && is_demotable(no_flags, dst_enc, src_enc); 13167 if (first_operand_demotable || second_operand_demotable) { 13168 if (size == EVEX_16bit) { 13169 emit_int8(0x66); 13170 } 13171 int src = first_operand_demotable ? src_enc : nds_enc; 13172 if (swap) { 13173 encode = is_prefixq ? prefixq_and_encode(dst_enc, src, is_map1) : prefix_and_encode(dst_enc, src, is_map1); 13174 } else { 13175 encode = is_prefixq ? prefixq_and_encode(src, dst_enc, is_map1) : prefix_and_encode(src, dst_enc, is_map1); 13176 } 13177 emit_opcode_prefix_and_encoding((unsigned char)opcode_byte, 0xC0, encode); 13178 } else { 13179 InstructionAttr attributes(AVX_128bit, is_prefixq, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13180 attributes.set_is_evex_instruction(); 13181 if (swap) { 13182 encode = vex_prefix_and_encode(nds_enc, dst_enc, src_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13183 } else { 13184 encode = vex_prefix_and_encode(src_enc, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13185 } 13186 emit_int16(opcode_byte, (0xC0 | encode)); 13187 } 13188 } 13189 13190 int Assembler::emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, 13191 InstructionAttr *attributes, bool no_flags, bool use_prefixq) { 13192 if (is_demotable(no_flags, dst_enc, nds_enc)) { 13193 if (pre == VEX_SIMD_66) { 13194 emit_int8(0x66); 13195 } 13196 return use_prefixq ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc); 13197 } 13198 attributes->set_is_evex_instruction(); 13199 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13200 } 13201 13202 int Assembler::emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, VexSimdPrefix pre, VexOpcode opc, 13203 InstructionAttr *attributes, bool no_flags, bool use_prefixq) { 13204 //Demote RegReg and RegRegImm instructions 13205 if (is_demotable(no_flags, dst_enc, nds_enc)) { 13206 return use_prefixq ? prefixq_and_encode(dst_enc) : prefix_and_encode(dst_enc); 13207 } 13208 attributes->set_is_evex_instruction(); 13209 return vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13210 } 13211 13212 int Assembler::emit_eevex_prefix_ndd(int dst_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) { 13213 attributes->set_is_evex_instruction(); 13214 return vex_prefix_and_encode(0, 0, dst_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13215 } 13216 13217 int Assembler::eevex_prefix_and_encode_nf(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, 13218 InstructionAttr *attributes, bool no_flags) { 13219 attributes->set_is_evex_instruction(); 13220 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ false, no_flags); 13221 } 13222 13223 void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc, 13224 int size, int op1, int op2, bool no_flags, bool is_commutative) { 13225 bool demotable = is_demotable(no_flags, dst->encoding(), src1->encoding()); 13226 if (!demotable && is_commutative) { 13227 if (is_demotable(no_flags, dst->encoding(), src2->encoding())) { 13228 // swap src1 and src2 13229 Register tmp = src1; 13230 src1 = src2; 13231 src2 = tmp; 13232 } 13233 } 13234 bool vex_w = (size == EVEX_64bit) ? true : false; 13235 bool use_prefixq = vex_w; 13236 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13237 // NDD shares its encoding bits with NDS bits for regular EVEX instruction. 13238 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine. 13239 (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags, use_prefixq); 13240 emit_arith(op1, op2, src1, src2); 13241 } 13242 13243 void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc, 13244 int size, int op1, int op2, bool no_flags) { 13245 int dst_enc = dst->encoding(); 13246 int nds_enc = nds->encoding(); 13247 bool demote = is_demotable(no_flags, dst_enc, nds_enc); 13248 if (demote) { 13249 (size == EVEX_64bit) ? (void) prefixq_and_encode(dst_enc) : (void) prefix_and_encode(dst_enc); 13250 } else { 13251 bool vex_w = (size == EVEX_64bit) ? true : false; 13252 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13253 attributes.set_is_evex_instruction(); 13254 vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags); 13255 13256 } 13257 emit_arith(op1, op2, nds, imm32, demote); 13258 } 13259 13260 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, 13261 VexOpcode opc, InstructionAttr *attributes) { 13262 if (UseAVX > 0) { 13263 int xreg_enc = xreg->encoding(); 13264 int nds_enc = nds->is_valid() ? nds->encoding() : 0; 13265 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes); 13266 } else { 13267 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding"); 13268 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w()); 13269 } 13270 } 13271 13272 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, 13273 VexOpcode opc, InstructionAttr *attributes, bool src_is_gpr) { 13274 int dst_enc = dst->encoding(); 13275 int src_enc = src->encoding(); 13276 if (UseAVX > 0) { 13277 int nds_enc = nds->is_valid() ? nds->encoding() : 0; 13278 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, src_is_gpr); 13279 } else { 13280 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding"); 13281 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w()); 13282 } 13283 } 13284 13285 bool Assembler::is_demotable(bool no_flags, int dst_enc, int nds_enc) { 13286 return (!no_flags && dst_enc == nds_enc); 13287 } 13288 13289 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 13290 assert(VM_Version::supports_avx(), ""); 13291 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13292 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 13293 emit_int16(0x5F, (0xC0 | encode)); 13294 } 13295 13296 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 13297 assert(VM_Version::supports_avx(), ""); 13298 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13299 attributes.set_rex_vex_w_reverted(); 13300 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13301 emit_int16(0x5F, (0xC0 | encode)); 13302 } 13303 13304 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 13305 assert(VM_Version::supports_avx(), ""); 13306 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13307 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 13308 emit_int16(0x5D, (0xC0 | encode)); 13309 } 13310 13311 void Assembler::eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 13312 assert(VM_Version::supports_avx10_2(), ""); 13313 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13314 attributes.set_is_evex_instruction(); 13315 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13316 emit_int24(0x53, (0xC0 | encode), imm8); 13317 } 13318 13319 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 13320 assert(VM_Version::supports_avx(), ""); 13321 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13322 attributes.set_rex_vex_w_reverted(); 13323 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13324 emit_int16(0x5D, (0xC0 | encode)); 13325 } 13326 13327 void Assembler::eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 13328 assert(VM_Version::supports_avx10_2(), ""); 13329 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13330 attributes.set_is_evex_instruction(); 13331 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13332 emit_int24(0x53, (0xC0 | encode), imm8); 13333 } 13334 13335 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) { 13336 assert(VM_Version::supports_avx(), ""); 13337 assert(vector_len <= AVX_256bit, ""); 13338 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 13339 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13340 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop)); 13341 } 13342 13343 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 13344 assert(VM_Version::supports_avx(), ""); 13345 assert(vector_len <= AVX_256bit, ""); 13346 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 13347 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13348 int src2_enc = src2->encoding(); 13349 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4)); 13350 } 13351 13352 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 13353 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), ""); 13354 assert(vector_len <= AVX_256bit, ""); 13355 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 13356 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13357 int src2_enc = src2->encoding(); 13358 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4)); 13359 } 13360 13361 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 13362 assert(VM_Version::supports_avx2(), ""); 13363 assert(vector_len <= AVX_256bit, ""); 13364 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 13365 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13366 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8); 13367 } 13368 13369 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) { 13370 assert(VM_Version::supports_avx(), ""); 13371 assert(vector_len <= AVX_256bit, ""); 13372 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 13373 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 13374 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison); 13375 } 13376 13377 void Assembler::evcmpph(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13378 ComparisonPredicateFP comparison, int vector_len) { 13379 assert(VM_Version::supports_avx512_fp16(), ""); 13380 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13381 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13382 attributes.set_is_evex_instruction(); 13383 attributes.set_embedded_opmask_register_specifier(mask); 13384 attributes.reset_is_clear_context(); 13385 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes); 13386 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 13387 } 13388 13389 void Assembler::evcmpsh(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicateFP comparison) { 13390 assert(VM_Version::supports_avx512_fp16(), ""); 13391 InstructionAttr attributes(Assembler::AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13392 attributes.set_is_evex_instruction(); 13393 attributes.set_embedded_opmask_register_specifier(mask); 13394 attributes.reset_is_clear_context(); 13395 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3A, &attributes); 13396 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 13397 } 13398 13399 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13400 ComparisonPredicateFP comparison, int vector_len) { 13401 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13402 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib 13403 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13404 attributes.set_is_evex_instruction(); 13405 attributes.set_embedded_opmask_register_specifier(mask); 13406 attributes.reset_is_clear_context(); 13407 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 13408 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 13409 } 13410 13411 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13412 ComparisonPredicateFP comparison, int vector_len) { 13413 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13414 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib 13415 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13416 attributes.set_is_evex_instruction(); 13417 attributes.set_embedded_opmask_register_specifier(mask); 13418 attributes.reset_is_clear_context(); 13419 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13420 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 13421 } 13422 13423 void Assembler::blendvps(XMMRegister dst, XMMRegister src) { 13424 assert(VM_Version::supports_sse4_1(), ""); 13425 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 13426 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13427 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13428 emit_int16(0x14, (0xC0 | encode)); 13429 } 13430 13431 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) { 13432 assert(VM_Version::supports_sse4_1(), ""); 13433 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 13434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13435 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13436 emit_int16(0x15, (0xC0 | encode)); 13437 } 13438 13439 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) { 13440 assert(VM_Version::supports_sse4_1(), ""); 13441 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 13442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13443 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13444 emit_int16(0x10, (0xC0 | encode)); 13445 } 13446 13447 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 13448 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), ""); 13449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13450 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13451 int src2_enc = src2->encoding(); 13452 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4)); 13453 } 13454 13455 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 13456 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13457 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13458 emit_int24(0x0C, (0xC0 | encode), imm8); 13459 } 13460 13461 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 13462 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 13463 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 13464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13465 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13466 emit_int16(0x64, (0xC0 | encode)); 13467 } 13468 13469 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 13470 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 13471 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 13472 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13473 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13474 emit_int16(0x65, (0xC0 | encode)); 13475 } 13476 13477 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 13478 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 13479 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 13480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13481 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13482 emit_int16(0x66, (0xC0 | encode)); 13483 } 13484 13485 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 13486 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 13487 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 13488 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13489 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13490 emit_int16(0x37, (0xC0 | encode)); 13491 } 13492 13493 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13494 int comparison, bool is_signed, int vector_len) { 13495 assert(VM_Version::supports_evex(), ""); 13496 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13497 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib 13498 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13499 attributes.set_is_evex_instruction(); 13500 attributes.set_embedded_opmask_register_specifier(mask); 13501 attributes.reset_is_clear_context(); 13502 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13503 int opcode = is_signed ? 0x1F : 0x1E; 13504 emit_int24(opcode, (0xC0 | encode), comparison); 13505 } 13506 13507 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 13508 int comparison, bool is_signed, int vector_len) { 13509 assert(VM_Version::supports_evex(), ""); 13510 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13511 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib 13512 InstructionMark im(this); 13513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13514 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 13515 attributes.set_is_evex_instruction(); 13516 attributes.set_embedded_opmask_register_specifier(mask); 13517 attributes.reset_is_clear_context(); 13518 int dst_enc = kdst->encoding(); 13519 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13520 int opcode = is_signed ? 0x1F : 0x1E; 13521 emit_int8((unsigned char)opcode); 13522 emit_operand(as_Register(dst_enc), src, 1); 13523 emit_int8((unsigned char)comparison); 13524 } 13525 13526 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13527 int comparison, bool is_signed, int vector_len) { 13528 assert(VM_Version::supports_evex(), ""); 13529 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13530 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib 13531 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13532 attributes.set_is_evex_instruction(); 13533 attributes.set_embedded_opmask_register_specifier(mask); 13534 attributes.reset_is_clear_context(); 13535 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13536 int opcode = is_signed ? 0x1F : 0x1E; 13537 emit_int24(opcode, (0xC0 | encode), comparison); 13538 } 13539 13540 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 13541 int comparison, bool is_signed, int vector_len) { 13542 assert(VM_Version::supports_evex(), ""); 13543 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13544 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib 13545 InstructionMark im(this); 13546 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13547 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 13548 attributes.set_is_evex_instruction(); 13549 attributes.set_embedded_opmask_register_specifier(mask); 13550 attributes.reset_is_clear_context(); 13551 int dst_enc = kdst->encoding(); 13552 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13553 int opcode = is_signed ? 0x1F : 0x1E; 13554 emit_int8((unsigned char)opcode); 13555 emit_operand(as_Register(dst_enc), src, 1); 13556 emit_int8((unsigned char)comparison); 13557 } 13558 13559 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13560 int comparison, bool is_signed, int vector_len) { 13561 assert(VM_Version::supports_evex(), ""); 13562 assert(VM_Version::supports_avx512bw(), ""); 13563 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13564 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib 13565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13566 attributes.set_is_evex_instruction(); 13567 attributes.set_embedded_opmask_register_specifier(mask); 13568 attributes.reset_is_clear_context(); 13569 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13570 int opcode = is_signed ? 0x3F : 0x3E; 13571 emit_int24(opcode, (0xC0 | encode), comparison); 13572 } 13573 13574 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 13575 int comparison, bool is_signed, int vector_len) { 13576 assert(VM_Version::supports_evex(), ""); 13577 assert(VM_Version::supports_avx512bw(), ""); 13578 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13579 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib 13580 InstructionMark im(this); 13581 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13582 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 13583 attributes.set_is_evex_instruction(); 13584 attributes.set_embedded_opmask_register_specifier(mask); 13585 attributes.reset_is_clear_context(); 13586 int dst_enc = kdst->encoding(); 13587 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13588 int opcode = is_signed ? 0x3F : 0x3E; 13589 emit_int8((unsigned char)opcode); 13590 emit_operand(as_Register(dst_enc), src, 1); 13591 emit_int8((unsigned char)comparison); 13592 } 13593 13594 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 13595 int comparison, bool is_signed, int vector_len) { 13596 assert(VM_Version::supports_evex(), ""); 13597 assert(VM_Version::supports_avx512bw(), ""); 13598 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13599 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib 13600 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13601 attributes.set_is_evex_instruction(); 13602 attributes.set_embedded_opmask_register_specifier(mask); 13603 attributes.reset_is_clear_context(); 13604 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13605 int opcode = is_signed ? 0x3F : 0x3E; 13606 emit_int24(opcode, (0xC0 | encode), comparison); 13607 } 13608 13609 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 13610 int comparison, bool is_signed, int vector_len) { 13611 assert(VM_Version::supports_evex(), ""); 13612 assert(VM_Version::supports_avx512bw(), ""); 13613 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 13614 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib 13615 InstructionMark im(this); 13616 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13617 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 13618 attributes.set_is_evex_instruction(); 13619 attributes.set_embedded_opmask_register_specifier(mask); 13620 attributes.reset_is_clear_context(); 13621 int dst_enc = kdst->encoding(); 13622 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13623 int opcode = is_signed ? 0x3F : 0x3E; 13624 emit_int8((unsigned char)opcode); 13625 emit_operand(as_Register(dst_enc), src, 1); 13626 emit_int8((unsigned char)comparison); 13627 } 13628 13629 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 13630 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13631 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13632 attributes.set_is_evex_instruction(); 13633 attributes.set_embedded_opmask_register_specifier(mask); 13634 if (merge) { 13635 attributes.reset_is_clear_context(); 13636 } 13637 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13638 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 13639 } 13640 13641 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 13642 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13643 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13644 attributes.set_is_evex_instruction(); 13645 attributes.set_embedded_opmask_register_specifier(mask); 13646 if (merge) { 13647 attributes.reset_is_clear_context(); 13648 } 13649 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13650 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 13651 } 13652 13653 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13654 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13655 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13656 attributes.set_is_evex_instruction(); 13657 attributes.set_embedded_opmask_register_specifier(mask); 13658 if (merge) { 13659 attributes.reset_is_clear_context(); 13660 } 13661 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13662 emit_int16(0x14, (0xC0 | encode)); 13663 } 13664 13665 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13666 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13667 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13668 attributes.set_is_evex_instruction(); 13669 attributes.set_embedded_opmask_register_specifier(mask); 13670 if (merge) { 13671 attributes.reset_is_clear_context(); 13672 } 13673 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13674 emit_int16(0x14, (0xC0 | encode)); 13675 } 13676 13677 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 13678 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13679 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13680 attributes.set_is_evex_instruction(); 13681 attributes.set_embedded_opmask_register_specifier(mask); 13682 if (merge) { 13683 attributes.reset_is_clear_context(); 13684 } 13685 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13686 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 13687 } 13688 13689 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 13690 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13691 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13692 attributes.set_is_evex_instruction(); 13693 attributes.set_embedded_opmask_register_specifier(mask); 13694 if (merge) { 13695 attributes.reset_is_clear_context(); 13696 } 13697 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13698 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 13699 } 13700 13701 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13702 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13703 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13704 attributes.set_is_evex_instruction(); 13705 attributes.set_embedded_opmask_register_specifier(mask); 13706 if (merge) { 13707 attributes.reset_is_clear_context(); 13708 } 13709 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13710 emit_int16(0x15, (0xC0 | encode)); 13711 } 13712 13713 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13714 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 13715 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13716 attributes.set_is_evex_instruction(); 13717 attributes.set_embedded_opmask_register_specifier(mask); 13718 if (merge) { 13719 attributes.reset_is_clear_context(); 13720 } 13721 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13722 emit_int16(0x15, (0xC0 | encode)); 13723 } 13724 13725 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) { 13726 assert(VM_Version::supports_avx(), ""); 13727 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13728 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 13729 int mask_enc = mask->encoding(); 13730 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4); 13731 } 13732 13733 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13734 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13735 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r 13736 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13737 attributes.set_is_evex_instruction(); 13738 attributes.set_embedded_opmask_register_specifier(mask); 13739 if (merge) { 13740 attributes.reset_is_clear_context(); 13741 } 13742 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13743 emit_int16(0x65, (0xC0 | encode)); 13744 } 13745 13746 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13747 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13748 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r 13749 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13750 attributes.set_is_evex_instruction(); 13751 attributes.set_embedded_opmask_register_specifier(mask); 13752 if (merge) { 13753 attributes.reset_is_clear_context(); 13754 } 13755 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13756 emit_int16(0x65, (0xC0 | encode)); 13757 } 13758 13759 void Assembler::evpblendmb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13760 assert(VM_Version::supports_avx512bw(), ""); 13761 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13762 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r 13763 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13764 attributes.set_is_evex_instruction(); 13765 attributes.set_embedded_opmask_register_specifier(mask); 13766 if (merge) { 13767 attributes.reset_is_clear_context(); 13768 } 13769 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13770 emit_int16(0x66, (0xC0 | encode)); 13771 } 13772 13773 void Assembler::evpblendmw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13774 assert(VM_Version::supports_avx512bw(), ""); 13775 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13776 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r 13777 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 13778 attributes.set_is_evex_instruction(); 13779 attributes.set_embedded_opmask_register_specifier(mask); 13780 if (merge) { 13781 attributes.reset_is_clear_context(); 13782 } 13783 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13784 emit_int16(0x66, (0xC0 | encode)); 13785 } 13786 13787 void Assembler::evpblendmd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13788 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13789 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r 13790 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13791 attributes.set_is_evex_instruction(); 13792 attributes.set_embedded_opmask_register_specifier(mask); 13793 if (merge) { 13794 attributes.reset_is_clear_context(); 13795 } 13796 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13797 emit_int16(0x64, (0xC0 | encode)); 13798 } 13799 13800 void Assembler::evpblendmq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 13801 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 13802 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r 13803 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 13804 attributes.set_is_evex_instruction(); 13805 attributes.set_embedded_opmask_register_specifier(mask); 13806 if (merge) { 13807 attributes.reset_is_clear_context(); 13808 } 13809 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13810 emit_int16(0x64, (0xC0 | encode)); 13811 } 13812 13813 void Assembler::bzhiq(Register dst, Register src1, Register src2) { 13814 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13815 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13816 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 13817 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13818 } 13819 13820 void Assembler::bzhil(Register dst, Register src1, Register src2) { 13821 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13822 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13823 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 13824 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13825 } 13826 13827 void Assembler::pextl(Register dst, Register src1, Register src2) { 13828 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13829 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13830 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true); 13831 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13832 } 13833 13834 void Assembler::pdepl(Register dst, Register src1, Register src2) { 13835 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13836 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13837 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true); 13838 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13839 } 13840 13841 void Assembler::pextq(Register dst, Register src1, Register src2) { 13842 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13843 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13844 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true); 13845 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13846 } 13847 13848 void Assembler::pdepq(Register dst, Register src1, Register src2) { 13849 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13850 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13851 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true); 13852 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 13853 } 13854 13855 void Assembler::pextl(Register dst, Register src1, Address src2) { 13856 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13857 InstructionMark im(this); 13858 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13859 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 13860 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 13861 emit_int8((unsigned char)0xF5); 13862 emit_operand(dst, src2, 0); 13863 } 13864 13865 void Assembler::pdepl(Register dst, Register src1, Address src2) { 13866 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13867 InstructionMark im(this); 13868 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13869 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 13870 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 13871 emit_int8((unsigned char)0xF5); 13872 emit_operand(dst, src2, 0); 13873 } 13874 13875 void Assembler::pextq(Register dst, Register src1, Address src2) { 13876 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13877 InstructionMark im(this); 13878 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13879 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 13880 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 13881 emit_int8((unsigned char)0xF5); 13882 emit_operand(dst, src2, 0); 13883 } 13884 13885 void Assembler::pdepq(Register dst, Register src1, Address src2) { 13886 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13887 InstructionMark im(this); 13888 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13889 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 13890 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 13891 emit_int8((unsigned char)0xF5); 13892 emit_operand(dst, src2, 0); 13893 } 13894 13895 void Assembler::sarxl(Register dst, Register src1, Register src2) { 13896 assert(VM_Version::supports_bmi2(), ""); 13897 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13898 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true); 13899 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13900 } 13901 13902 void Assembler::sarxl(Register dst, Address src1, Register src2) { 13903 assert(VM_Version::supports_bmi2(), ""); 13904 InstructionMark im(this); 13905 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13906 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 13907 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 13908 emit_int8((unsigned char)0xF7); 13909 emit_operand(dst, src1, 0); 13910 } 13911 13912 void Assembler::sarxq(Register dst, Register src1, Register src2) { 13913 assert(VM_Version::supports_bmi2(), ""); 13914 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13915 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true); 13916 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13917 } 13918 13919 void Assembler::sarxq(Register dst, Address src1, Register src2) { 13920 assert(VM_Version::supports_bmi2(), ""); 13921 InstructionMark im(this); 13922 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13923 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 13924 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 13925 emit_int8((unsigned char)0xF7); 13926 emit_operand(dst, src1, 0); 13927 } 13928 13929 void Assembler::shlxl(Register dst, Register src1, Register src2) { 13930 assert(VM_Version::supports_bmi2(), ""); 13931 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13932 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true); 13933 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13934 } 13935 13936 void Assembler::shlxl(Register dst, Address src1, Register src2) { 13937 assert(VM_Version::supports_bmi2(), ""); 13938 InstructionMark im(this); 13939 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13940 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 13941 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13942 emit_int8((unsigned char)0xF7); 13943 emit_operand(dst, src1, 0); 13944 } 13945 13946 void Assembler::shlxq(Register dst, Register src1, Register src2) { 13947 assert(VM_Version::supports_bmi2(), ""); 13948 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13949 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true); 13950 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13951 } 13952 13953 void Assembler::shlxq(Register dst, Address src1, Register src2) { 13954 assert(VM_Version::supports_bmi2(), ""); 13955 InstructionMark im(this); 13956 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13957 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 13958 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 13959 emit_int8((unsigned char)0xF7); 13960 emit_operand(dst, src1, 0); 13961 } 13962 13963 void Assembler::shrxl(Register dst, Register src1, Register src2) { 13964 assert(VM_Version::supports_bmi2(), ""); 13965 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13966 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true); 13967 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13968 } 13969 13970 void Assembler::shrxl(Register dst, Address src1, Register src2) { 13971 assert(VM_Version::supports_bmi2(), ""); 13972 InstructionMark im(this); 13973 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13974 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 13975 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 13976 emit_int8((unsigned char)0xF7); 13977 emit_operand(dst, src1, 0); 13978 } 13979 13980 void Assembler::shrxq(Register dst, Register src1, Register src2) { 13981 assert(VM_Version::supports_bmi2(), ""); 13982 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13983 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true); 13984 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13985 } 13986 13987 void Assembler::shrxq(Register dst, Address src1, Register src2) { 13988 assert(VM_Version::supports_bmi2(), ""); 13989 InstructionMark im(this); 13990 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13991 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 13992 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 13993 emit_int8((unsigned char)0xF7); 13994 emit_operand(dst, src1, 0); 13995 } 13996 13997 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) { 13998 assert(VM_Version::supports_avx512vldq(), ""); 13999 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14000 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14001 attributes.set_is_evex_instruction(); 14002 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14003 emit_int16(0x39, (0xC0 | encode)); 14004 } 14005 14006 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) { 14007 assert(VM_Version::supports_avx512vldq(), ""); 14008 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14009 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14010 attributes.set_is_evex_instruction(); 14011 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14012 emit_int16(0x39, (0xC0 | encode)); 14013 } 14014 14015 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) { 14016 assert(VM_Version::supports_avx512vlbw(), ""); 14017 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14018 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14019 attributes.set_is_evex_instruction(); 14020 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14021 emit_int16(0x29, (0xC0 | encode)); 14022 } 14023 14024 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) { 14025 assert(VM_Version::supports_avx512vlbw(), ""); 14026 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14027 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14028 attributes.set_is_evex_instruction(); 14029 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14030 emit_int16(0x29, (0xC0 | encode)); 14031 } 14032 14033 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) { 14034 assert(VM_Version::supports_avx512vldq(), ""); 14035 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14036 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14037 attributes.set_is_evex_instruction(); 14038 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14039 emit_int16(0x38, (0xC0 | encode)); 14040 } 14041 14042 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) { 14043 assert(VM_Version::supports_avx512vldq(), ""); 14044 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14045 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14046 attributes.set_is_evex_instruction(); 14047 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14048 emit_int16(0x38, (0xC0 | encode)); 14049 } 14050 14051 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) { 14052 assert(VM_Version::supports_avx512vlbw(), ""); 14053 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14054 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14055 attributes.set_is_evex_instruction(); 14056 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14057 emit_int16(0x28, (0xC0 | encode)); 14058 } 14059 14060 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) { 14061 assert(VM_Version::supports_avx512vlbw(), ""); 14062 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, ""); 14063 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 14064 attributes.set_is_evex_instruction(); 14065 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 14066 emit_int16(0x28, (0xC0 | encode)); 14067 } 14068 14069 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14070 assert(VM_Version::supports_avx512_vbmi2(), ""); 14071 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14072 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14073 attributes.set_embedded_opmask_register_specifier(mask); 14074 attributes.set_is_evex_instruction(); 14075 if (merge) { 14076 attributes.reset_is_clear_context(); 14077 } 14078 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14079 emit_int16((unsigned char)0x63, (0xC0 | encode)); 14080 } 14081 14082 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14083 assert(VM_Version::supports_avx512_vbmi2(), ""); 14084 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14085 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14086 attributes.set_embedded_opmask_register_specifier(mask); 14087 attributes.set_is_evex_instruction(); 14088 if (merge) { 14089 attributes.reset_is_clear_context(); 14090 } 14091 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14092 emit_int16((unsigned char)0x63, (0xC0 | encode)); 14093 } 14094 14095 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14096 assert(VM_Version::supports_evex(), ""); 14097 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14098 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14099 attributes.set_embedded_opmask_register_specifier(mask); 14100 attributes.set_is_evex_instruction(); 14101 if (merge) { 14102 attributes.reset_is_clear_context(); 14103 } 14104 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14105 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 14106 } 14107 14108 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14109 assert(VM_Version::supports_evex(), ""); 14110 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14111 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14112 attributes.set_embedded_opmask_register_specifier(mask); 14113 attributes.set_is_evex_instruction(); 14114 if (merge) { 14115 attributes.reset_is_clear_context(); 14116 } 14117 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14118 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 14119 } 14120 14121 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14122 assert(VM_Version::supports_evex(), ""); 14123 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14124 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14125 attributes.set_embedded_opmask_register_specifier(mask); 14126 attributes.set_is_evex_instruction(); 14127 if (merge) { 14128 attributes.reset_is_clear_context(); 14129 } 14130 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14131 emit_int16((unsigned char)0x8A, (0xC0 | encode)); 14132 } 14133 14134 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 14135 assert(VM_Version::supports_evex(), ""); 14136 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 14137 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 14138 attributes.set_embedded_opmask_register_specifier(mask); 14139 attributes.set_is_evex_instruction(); 14140 if (merge) { 14141 attributes.reset_is_clear_context(); 14142 } 14143 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 14144 emit_int16((unsigned char)0x8A, (0xC0 | encode)); 14145 } 14146 14147 // This should only be used by 64bit instructions that can use rip-relative 14148 // it cannot be used by instructions that want an immediate value. 14149 14150 // Determine whether an address is always reachable in rip-relative addressing mode 14151 // when accessed from the code cache. 14152 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) { 14153 switch (reloc_type) { 14154 // This should be rip-relative and easily reachable. 14155 case relocInfo::internal_word_type: { 14156 return true; 14157 } 14158 // This should be rip-relative within the code cache and easily 14159 // reachable until we get huge code caches. (At which point 14160 // IC code is going to have issues). 14161 case relocInfo::virtual_call_type: 14162 case relocInfo::opt_virtual_call_type: 14163 case relocInfo::static_call_type: 14164 case relocInfo::static_stub_type: { 14165 return true; 14166 } 14167 case relocInfo::runtime_call_type: 14168 case relocInfo::external_word_type: 14169 case relocInfo::poll_return_type: // these are really external_word but need special 14170 case relocInfo::poll_type: { // relocs to identify them 14171 return CodeCache::contains(target); 14172 } 14173 default: { 14174 return false; 14175 } 14176 } 14177 } 14178 14179 // Determine whether an address is reachable in rip-relative addressing mode from the code cache. 14180 static bool is_reachable(address target, relocInfo::relocType reloc_type) { 14181 if (is_always_reachable(target, reloc_type)) { 14182 return true; 14183 } 14184 switch (reloc_type) { 14185 // None will force a 64bit literal to the code stream. Likely a placeholder 14186 // for something that will be patched later and we need to certain it will 14187 // always be reachable. 14188 case relocInfo::none: { 14189 return false; 14190 } 14191 case relocInfo::runtime_call_type: 14192 case relocInfo::external_word_type: 14193 case relocInfo::poll_return_type: // these are really external_word but need special 14194 case relocInfo::poll_type: { // relocs to identify them 14195 assert(!CodeCache::contains(target), "always reachable"); 14196 if (ForceUnreachable) { 14197 return false; // stress the correction code 14198 } 14199 // For external_word_type/runtime_call_type if it is reachable from where we 14200 // are now (possibly a temp buffer) and where we might end up 14201 // anywhere in the code cache then we are always reachable. 14202 // This would have to change if we ever save/restore shared code to be more pessimistic. 14203 // Code buffer has to be allocated in the code cache, so check against 14204 // code cache boundaries cover that case. 14205 // 14206 // In rip-relative addressing mode, an effective address is formed by adding displacement 14207 // to the 64-bit RIP of the next instruction which is not known yet. Considering target address 14208 // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough 14209 // to account for that. 14210 return Assembler::is_simm32(target - CodeCache::low_bound()) && 14211 Assembler::is_simm32(target - CodeCache::high_bound()); 14212 } 14213 default: { 14214 return false; 14215 } 14216 } 14217 } 14218 14219 bool Assembler::reachable(AddressLiteral adr) { 14220 assert(CodeCache::contains(pc()), "required"); 14221 if (adr.is_lval()) { 14222 return false; 14223 } 14224 return is_reachable(adr.target(), adr.reloc()); 14225 } 14226 14227 bool Assembler::always_reachable(AddressLiteral adr) { 14228 assert(CodeCache::contains(pc()), "required"); 14229 if (adr.is_lval()) { 14230 return false; 14231 } 14232 return is_always_reachable(adr.target(), adr.reloc()); 14233 } 14234 14235 void Assembler::emit_data64(jlong data, 14236 relocInfo::relocType rtype, 14237 int format) { 14238 if (rtype == relocInfo::none) { 14239 emit_int64(data); 14240 } else { 14241 emit_data64(data, Relocation::spec_simple(rtype), format); 14242 } 14243 } 14244 14245 void Assembler::emit_data64(jlong data, 14246 RelocationHolder const& rspec, 14247 int format) { 14248 assert(imm_operand == 0, "default format must be immediate in this file"); 14249 assert(imm_operand == format, "must be immediate"); 14250 assert(inst_mark() != nullptr, "must be inside InstructionMark"); 14251 // Do not use AbstractAssembler::relocate, which is not intended for 14252 // embedded words. Instead, relocate to the enclosing instruction. 14253 code_section()->relocate(inst_mark(), rspec, format); 14254 #ifdef ASSERT 14255 check_relocation(rspec, format); 14256 #endif 14257 emit_int64(data); 14258 } 14259 14260 int Assembler::get_base_prefix_bits(int enc) { 14261 int bits = 0; 14262 if (enc & 16) bits |= REX2BIT_B4; 14263 if (enc & 8) bits |= REX2BIT_B; 14264 return bits; 14265 } 14266 14267 int Assembler::get_index_prefix_bits(int enc) { 14268 int bits = 0; 14269 if (enc & 16) bits |= REX2BIT_X4; 14270 if (enc & 8) bits |= REX2BIT_X; 14271 return bits; 14272 } 14273 14274 int Assembler::get_base_prefix_bits(Register base) { 14275 return base->is_valid() ? get_base_prefix_bits(base->encoding()) : 0; 14276 } 14277 14278 int Assembler::get_index_prefix_bits(Register index) { 14279 return index->is_valid() ? get_index_prefix_bits(index->encoding()) : 0; 14280 } 14281 14282 int Assembler::get_reg_prefix_bits(int enc) { 14283 int bits = 0; 14284 if (enc & 16) bits |= REX2BIT_R4; 14285 if (enc & 8) bits |= REX2BIT_R; 14286 return bits; 14287 } 14288 14289 void Assembler::prefix(Register reg) { 14290 if (reg->encoding() >= 16) { 14291 prefix16(WREX2 | get_base_prefix_bits(reg->encoding())); 14292 } else if (reg->encoding() >= 8) { 14293 prefix(REX_B); 14294 } 14295 } 14296 14297 void Assembler::prefix(Register dst, Register src, Prefix p) { 14298 if ((p & WREX2) || src->encoding() >= 16 || dst->encoding() >= 16) { 14299 prefix_rex2(dst, src); 14300 return; 14301 } 14302 if (src->encoding() >= 8) { 14303 p = (Prefix)(p | REX_B); 14304 } 14305 if (dst->encoding() >= 8) { 14306 p = (Prefix)(p | REX_R); 14307 } 14308 if (p != Prefix_EMPTY) { 14309 // do not generate an empty prefix 14310 prefix(p); 14311 } 14312 } 14313 14314 void Assembler::prefix_rex2(Register dst, Register src) { 14315 int bits = 0; 14316 bits |= get_base_prefix_bits(src->encoding()); 14317 bits |= get_reg_prefix_bits(dst->encoding()); 14318 prefix16(WREX2 | bits); 14319 } 14320 14321 void Assembler::prefix(Register dst, Address adr, Prefix p) { 14322 if (adr.base_needs_rex2() || adr.index_needs_rex2() || dst->encoding() >= 16) { 14323 prefix_rex2(dst, adr); 14324 } 14325 if (adr.base_needs_rex()) { 14326 if (adr.index_needs_rex()) { 14327 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X"); 14328 } else { 14329 p = (Prefix)(p | REX_B); 14330 } 14331 } else { 14332 if (adr.index_needs_rex()) { 14333 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X"); 14334 } 14335 } 14336 if (dst->encoding() >= 8) { 14337 p = (Prefix)(p | REX_R); 14338 } 14339 if (p != Prefix_EMPTY) { 14340 // do not generate an empty prefix 14341 prefix(p); 14342 } 14343 } 14344 14345 void Assembler::prefix_rex2(Register dst, Address adr) { 14346 assert(!adr.index_needs_rex2(), "prefix(Register dst, Address adr) does not support handling of an X"); 14347 int bits = 0; 14348 bits |= get_base_prefix_bits(adr.base()); 14349 bits |= get_reg_prefix_bits(dst->encoding()); 14350 prefix16(WREX2 | bits); 14351 } 14352 14353 void Assembler::prefix(Address adr, bool is_map1) { 14354 if (adr.base_needs_rex2() || adr.index_needs_rex2()) { 14355 prefix_rex2(adr, is_map1); 14356 return; 14357 } 14358 if (adr.base_needs_rex()) { 14359 if (adr.index_needs_rex()) { 14360 prefix(REX_XB); 14361 } else { 14362 prefix(REX_B); 14363 } 14364 } else { 14365 if (adr.index_needs_rex()) { 14366 prefix(REX_X); 14367 } 14368 } 14369 if (is_map1) emit_int8(0x0F); 14370 } 14371 14372 void Assembler::prefix_rex2(Address adr, bool is_map1) { 14373 int bits = is_map1 ? REX2BIT_M0 : 0; 14374 bits |= get_base_prefix_bits(adr.base()); 14375 bits |= get_index_prefix_bits(adr.index()); 14376 prefix16(WREX2 | bits); 14377 } 14378 14379 void Assembler::prefix(Address adr, Register reg, bool byteinst, bool is_map1) { 14380 if (reg->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) { 14381 prefix_rex2(adr, reg, byteinst, is_map1); 14382 return; 14383 } 14384 if (reg->encoding() < 8) { 14385 if (adr.base_needs_rex()) { 14386 if (adr.index_needs_rex()) { 14387 prefix(REX_XB); 14388 } else { 14389 prefix(REX_B); 14390 } 14391 } else { 14392 if (adr.index_needs_rex()) { 14393 prefix(REX_X); 14394 } else if (byteinst && reg->encoding() >= 4) { 14395 prefix(REX); 14396 } 14397 } 14398 } else { 14399 if (adr.base_needs_rex()) { 14400 if (adr.index_needs_rex()) { 14401 prefix(REX_RXB); 14402 } else { 14403 prefix(REX_RB); 14404 } 14405 } else { 14406 if (adr.index_needs_rex()) { 14407 prefix(REX_RX); 14408 } else { 14409 prefix(REX_R); 14410 } 14411 } 14412 } 14413 if (is_map1) emit_int8(0x0F); 14414 } 14415 14416 void Assembler::prefix_rex2(Address adr, Register reg, bool byteinst, bool is_map1) { 14417 int bits = is_map1 ? REX2BIT_M0 : 0; 14418 bits |= get_base_prefix_bits(adr.base()); 14419 bits |= get_index_prefix_bits(adr.index()); 14420 bits |= get_reg_prefix_bits(reg->encoding()); 14421 prefix16(WREX2 | bits); 14422 } 14423 14424 void Assembler::prefix(Address adr, XMMRegister reg) { 14425 if (reg->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) { 14426 prefixq_rex2(adr, reg); 14427 return; 14428 } 14429 if (reg->encoding() < 8) { 14430 if (adr.base_needs_rex()) { 14431 if (adr.index_needs_rex()) { 14432 prefix(REX_XB); 14433 } else { 14434 prefix(REX_B); 14435 } 14436 } else { 14437 if (adr.index_needs_rex()) { 14438 prefix(REX_X); 14439 } 14440 } 14441 } else { 14442 if (adr.base_needs_rex()) { 14443 if (adr.index_needs_rex()) { 14444 prefix(REX_RXB); 14445 } else { 14446 prefix(REX_RB); 14447 } 14448 } else { 14449 if (adr.index_needs_rex()) { 14450 prefix(REX_RX); 14451 } else { 14452 prefix(REX_R); 14453 } 14454 } 14455 } 14456 } 14457 14458 void Assembler::prefix_rex2(Address adr, XMMRegister src) { 14459 int bits = 0; 14460 bits |= get_base_prefix_bits(adr.base()); 14461 bits |= get_index_prefix_bits(adr.index()); 14462 bits |= get_reg_prefix_bits(src->encoding()); 14463 prefix16(WREX2 | bits); 14464 } 14465 14466 int Assembler::prefix_and_encode(int reg_enc, bool byteinst, bool is_map1) { 14467 if (reg_enc >= 16) { 14468 return prefix_and_encode_rex2(reg_enc, is_map1); 14469 } 14470 if (reg_enc >= 8) { 14471 prefix(REX_B); 14472 reg_enc -= 8; 14473 } else if (byteinst && reg_enc >= 4) { 14474 prefix(REX); 14475 } 14476 int opc_prefix = is_map1 ? 0x0F00 : 0; 14477 return opc_prefix | reg_enc; 14478 } 14479 14480 int Assembler::prefix_and_encode_rex2(int reg_enc, bool is_map1) { 14481 prefix16(WREX2 | (is_map1 ? REX2BIT_M0 : 0) | get_base_prefix_bits(reg_enc)); 14482 return reg_enc & 0x7; 14483 } 14484 14485 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte, bool is_map1) { 14486 if (src_enc >= 16 || dst_enc >= 16) { 14487 return prefix_and_encode_rex2(dst_enc, src_enc, is_map1 ? REX2BIT_M0 : 0); 14488 } 14489 if (dst_enc < 8) { 14490 if (src_enc >= 8) { 14491 prefix(REX_B); 14492 src_enc -= 8; 14493 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) { 14494 prefix(REX); 14495 } 14496 } else { 14497 if (src_enc < 8) { 14498 prefix(REX_R); 14499 } else { 14500 prefix(REX_RB); 14501 src_enc -= 8; 14502 } 14503 dst_enc -= 8; 14504 } 14505 int opcode_prefix = is_map1 ? 0x0F00 : 0; 14506 return opcode_prefix | (dst_enc << 3 | src_enc); 14507 } 14508 14509 int Assembler::prefix_and_encode_rex2(int dst_enc, int src_enc, int init_bits) { 14510 int bits = init_bits; 14511 bits |= get_reg_prefix_bits(dst_enc); 14512 bits |= get_base_prefix_bits(src_enc); 14513 dst_enc &= 0x7; 14514 src_enc &= 0x7; 14515 prefix16(WREX2 | bits); 14516 return dst_enc << 3 | src_enc; 14517 } 14518 14519 bool Assembler::prefix_is_rex2(int prefix) { 14520 return (prefix & 0xFF00) == WREX2; 14521 } 14522 14523 int Assembler::get_prefixq_rex2(Address adr, bool is_map1) { 14524 assert(UseAPX, "APX features not enabled"); 14525 int bits = REX2BIT_W; 14526 if (is_map1) bits |= REX2BIT_M0; 14527 bits |= get_base_prefix_bits(adr.base()); 14528 bits |= get_index_prefix_bits(adr.index()); 14529 return WREX2 | bits; 14530 } 14531 14532 int Assembler::get_prefixq(Address adr, bool is_map1) { 14533 if (adr.base_needs_rex2() || adr.index_needs_rex2()) { 14534 return get_prefixq_rex2(adr, is_map1); 14535 } 14536 int8_t prfx = get_prefixq(adr, rax); 14537 assert(REX_W <= prfx && prfx <= REX_WXB, "must be"); 14538 return is_map1 ? (((int16_t)prfx) << 8) | 0x0F : (int16_t)prfx; 14539 } 14540 14541 int Assembler::get_prefixq(Address adr, Register src, bool is_map1) { 14542 if (adr.base_needs_rex2() || adr.index_needs_rex2() || src->encoding() >= 16) { 14543 return get_prefixq_rex2(adr, src, is_map1); 14544 } 14545 int8_t prfx = (int8_t)(REX_W + 14546 ((int)adr.base_needs_rex()) + 14547 ((int)adr.index_needs_rex() << 1) + 14548 ((int)(src->encoding() >= 8) << 2)); 14549 #ifdef ASSERT 14550 if (src->encoding() < 8) { 14551 if (adr.base_needs_rex()) { 14552 if (adr.index_needs_rex()) { 14553 assert(prfx == REX_WXB, "must be"); 14554 } else { 14555 assert(prfx == REX_WB, "must be"); 14556 } 14557 } else { 14558 if (adr.index_needs_rex()) { 14559 assert(prfx == REX_WX, "must be"); 14560 } else { 14561 assert(prfx == REX_W, "must be"); 14562 } 14563 } 14564 } else { 14565 if (adr.base_needs_rex()) { 14566 if (adr.index_needs_rex()) { 14567 assert(prfx == REX_WRXB, "must be"); 14568 } else { 14569 assert(prfx == REX_WRB, "must be"); 14570 } 14571 } else { 14572 if (adr.index_needs_rex()) { 14573 assert(prfx == REX_WRX, "must be"); 14574 } else { 14575 assert(prfx == REX_WR, "must be"); 14576 } 14577 } 14578 } 14579 #endif 14580 return is_map1 ? (((int16_t)prfx) << 8) | 0x0F : (int16_t)prfx; 14581 } 14582 14583 int Assembler::get_prefixq_rex2(Address adr, Register src, bool is_map1) { 14584 assert(UseAPX, "APX features not enabled"); 14585 int bits = REX2BIT_W; 14586 if (is_map1) bits |= REX2BIT_M0; 14587 bits |= get_base_prefix_bits(adr.base()); 14588 bits |= get_index_prefix_bits(adr.index()); 14589 bits |= get_reg_prefix_bits(src->encoding()); 14590 return WREX2 | bits; 14591 } 14592 14593 void Assembler::prefixq(Address adr) { 14594 if (adr.base_needs_rex2() || adr.index_needs_rex2()) { 14595 prefix16(get_prefixq_rex2(adr)); 14596 } else { 14597 emit_int8(get_prefixq(adr)); 14598 } 14599 } 14600 14601 void Assembler::prefixq(Address adr, Register src, bool is_map1) { 14602 if (adr.base_needs_rex2() || adr.index_needs_rex2() || src->encoding() >= 16) { 14603 prefix16(get_prefixq_rex2(adr, src, is_map1)); 14604 } else { 14605 emit_int8(get_prefixq(adr, src)); 14606 if (is_map1) emit_int8(0x0F); 14607 } 14608 } 14609 14610 14611 void Assembler::prefixq(Address adr, XMMRegister src) { 14612 if (src->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) { 14613 prefixq_rex2(adr, src); 14614 return; 14615 } 14616 if (src->encoding() < 8) { 14617 if (adr.base_needs_rex()) { 14618 if (adr.index_needs_rex()) { 14619 prefix(REX_WXB); 14620 } else { 14621 prefix(REX_WB); 14622 } 14623 } else { 14624 if (adr.index_needs_rex()) { 14625 prefix(REX_WX); 14626 } else { 14627 prefix(REX_W); 14628 } 14629 } 14630 } else { 14631 if (adr.base_needs_rex()) { 14632 if (adr.index_needs_rex()) { 14633 prefix(REX_WRXB); 14634 } else { 14635 prefix(REX_WRB); 14636 } 14637 } else { 14638 if (adr.index_needs_rex()) { 14639 prefix(REX_WRX); 14640 } else { 14641 prefix(REX_WR); 14642 } 14643 } 14644 } 14645 } 14646 14647 void Assembler::prefixq_rex2(Address adr, XMMRegister src) { 14648 int bits = REX2BIT_W; 14649 bits |= get_base_prefix_bits(adr.base()); 14650 bits |= get_index_prefix_bits(adr.index()); 14651 bits |= get_reg_prefix_bits(src->encoding()); 14652 prefix16(WREX2 | bits); 14653 } 14654 14655 int Assembler::prefixq_and_encode(int reg_enc, bool is_map1) { 14656 if (reg_enc >= 16) { 14657 return prefixq_and_encode_rex2(reg_enc, is_map1); 14658 } 14659 if (reg_enc < 8) { 14660 prefix(REX_W); 14661 } else { 14662 prefix(REX_WB); 14663 reg_enc -= 8; 14664 } 14665 int opcode_prefix = is_map1 ? 0x0F00 : 0; 14666 return opcode_prefix | reg_enc; 14667 } 14668 14669 14670 int Assembler::prefixq_and_encode_rex2(int reg_enc, bool is_map1) { 14671 prefix16(WREX2 | REX2BIT_W | (is_map1 ? REX2BIT_M0: 0) | get_base_prefix_bits(reg_enc)); 14672 return reg_enc & 0x7; 14673 } 14674 14675 int Assembler::prefixq_and_encode(int dst_enc, int src_enc, bool is_map1) { 14676 if (dst_enc >= 16 || src_enc >= 16) { 14677 return prefixq_and_encode_rex2(dst_enc, src_enc, is_map1); 14678 } 14679 if (dst_enc < 8) { 14680 if (src_enc < 8) { 14681 prefix(REX_W); 14682 } else { 14683 prefix(REX_WB); 14684 src_enc -= 8; 14685 } 14686 } else { 14687 if (src_enc < 8) { 14688 prefix(REX_WR); 14689 } else { 14690 prefix(REX_WRB); 14691 src_enc -= 8; 14692 } 14693 dst_enc -= 8; 14694 } 14695 int opcode_prefix = is_map1 ? 0x0F00 : 0; 14696 return opcode_prefix | (dst_enc << 3 | src_enc); 14697 } 14698 14699 int Assembler::prefixq_and_encode_rex2(int dst_enc, int src_enc, bool is_map1) { 14700 int init_bits = REX2BIT_W | (is_map1 ? REX2BIT_M0 : 0); 14701 return prefix_and_encode_rex2(dst_enc, src_enc, init_bits); 14702 } 14703 14704 void Assembler::emit_prefix_and_int8(int prefix, int b1) { 14705 if ((prefix & 0xFF00) == 0) { 14706 emit_int16(prefix, b1); 14707 } else { 14708 assert((prefix & 0xFF00) != WREX2 || UseAPX, "APX features not enabled"); 14709 emit_int24((prefix & 0xFF00) >> 8, prefix & 0x00FF, b1); 14710 } 14711 } 14712 14713 void Assembler::adcq(Register dst, int32_t imm32) { 14714 (void) prefixq_and_encode(dst->encoding()); 14715 emit_arith(0x81, 0xD0, dst, imm32); 14716 } 14717 14718 void Assembler::adcq(Register dst, Address src) { 14719 InstructionMark im(this); 14720 emit_prefix_and_int8(get_prefixq(src, dst), 0x13); 14721 emit_operand(dst, src, 0); 14722 } 14723 14724 void Assembler::adcq(Register dst, Register src) { 14725 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 14726 emit_arith(0x13, 0xC0, dst, src); 14727 } 14728 14729 void Assembler::addq(Address dst, int32_t imm32) { 14730 InstructionMark im(this); 14731 prefixq(dst); 14732 emit_arith_operand(0x81, rax, dst, imm32); 14733 } 14734 14735 void Assembler::eaddq(Register dst, Address src, int32_t imm32, bool no_flags) { 14736 InstructionMark im(this); 14737 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14738 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14739 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 14740 emit_arith_operand(0x81, rax, src, imm32); 14741 } 14742 14743 void Assembler::addq(Address dst, Register src) { 14744 InstructionMark im(this); 14745 emit_prefix_and_int8(get_prefixq(dst, src), 0x01); 14746 emit_operand(src, dst, 0); 14747 } 14748 14749 void Assembler::eaddq(Register dst, Address src1, Register src2, bool no_flags) { 14750 InstructionMark im(this); 14751 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */); 14752 } 14753 14754 void Assembler::addq(Register dst, int32_t imm32) { 14755 (void) prefixq_and_encode(dst->encoding()); 14756 emit_arith(0x81, 0xC0, dst, imm32); 14757 } 14758 14759 void Assembler::eaddq(Register dst, Register src, int32_t imm32, bool no_flags) { 14760 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xC0, no_flags); 14761 } 14762 14763 void Assembler::addq(Register dst, Address src) { 14764 InstructionMark im(this); 14765 emit_prefix_and_int8(get_prefixq(src, dst), 0x03); 14766 emit_operand(dst, src, 0); 14767 } 14768 14769 void Assembler::eaddq(Register dst, Register src1, Address src2, bool no_flags) { 14770 InstructionMark im(this); 14771 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, no_flags); 14772 } 14773 14774 void Assembler::addq(Register dst, Register src) { 14775 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 14776 emit_arith(0x03, 0xC0, dst, src); 14777 } 14778 14779 void Assembler::eaddq(Register dst, Register src1, Register src2, bool no_flags) { 14780 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, 0xC0, no_flags, true /* is_commutative */); 14781 } 14782 14783 void Assembler::adcxq(Register dst, Register src) { 14784 //assert(VM_Version::supports_adx(), "adx instructions not supported"); 14785 if (needs_rex2(dst, src)) { 14786 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14787 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true); 14788 emit_int16((unsigned char)0x66, (0xC0 | encode)); 14789 } else { 14790 emit_int8(0x66); 14791 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 14792 emit_int32(0x0F, 14793 0x38, 14794 (unsigned char)0xF6, 14795 (0xC0 | encode)); 14796 } 14797 } 14798 14799 void Assembler::eadcxq(Register dst, Register src1, Register src2) { 14800 if (is_demotable(false, dst->encoding(), src1->encoding())) { 14801 return adcxq(dst, src2); 14802 } 14803 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14804 int encode = emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */); 14805 emit_int16((unsigned char)0x66, (0xC0 | encode)); 14806 } 14807 14808 void Assembler::adoxq(Register dst, Register src) { 14809 //assert(VM_Version::supports_adx(), "adx instructions not supported"); 14810 if (needs_rex2(dst, src)) { 14811 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14812 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true); 14813 emit_int16((unsigned char)0x66, (0xC0 | encode)); 14814 } else { 14815 emit_int8((unsigned char)0xF3); 14816 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 14817 emit_int32(0x0F, 14818 0x38, 14819 (unsigned char)0xF6, 14820 (0xC0 | encode)); 14821 } 14822 } 14823 14824 void Assembler::eadoxq(Register dst, Register src1, Register src2) { 14825 if (is_demotable(false, dst->encoding(), src1->encoding())) { 14826 return adoxq(dst, src2); 14827 } 14828 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14829 int encode = emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */); 14830 emit_int16((unsigned char)0x66, (0xC0 | encode)); 14831 } 14832 14833 void Assembler::andq(Address dst, int32_t imm32) { 14834 InstructionMark im(this); 14835 prefixq(dst); 14836 emit_arith_operand(0x81, as_Register(4), dst, imm32); 14837 } 14838 14839 void Assembler::eandq(Register dst, Address src, int32_t imm32, bool no_flags) { 14840 InstructionMark im(this); 14841 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14842 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14843 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 14844 emit_arith_operand(0x81, as_Register(4), src, imm32); 14845 } 14846 14847 void Assembler::andq(Register dst, int32_t imm32) { 14848 (void) prefixq_and_encode(dst->encoding()); 14849 emit_arith(0x81, 0xE0, dst, imm32); 14850 } 14851 14852 void Assembler::eandq(Register dst, Register src, int32_t imm32, bool no_flags) { 14853 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xE0, no_flags); 14854 } 14855 14856 void Assembler::andq(Register dst, Address src) { 14857 InstructionMark im(this); 14858 emit_prefix_and_int8(get_prefixq(src, dst), 0x23); 14859 emit_operand(dst, src, 0); 14860 } 14861 14862 void Assembler::eandq(Register dst, Register src1, Address src2, bool no_flags) { 14863 InstructionMark im(this); 14864 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, no_flags); 14865 } 14866 14867 void Assembler::andq(Register dst, Register src) { 14868 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 14869 emit_arith(0x23, 0xC0, dst, src); 14870 } 14871 14872 void Assembler::eandq(Register dst, Register src1, Register src2, bool no_flags) { 14873 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, 0xC0, no_flags, true /* is_commutative */); 14874 } 14875 14876 void Assembler::andq(Address dst, Register src) { 14877 InstructionMark im(this); 14878 emit_prefix_and_int8(get_prefixq(dst, src), 0x21); 14879 emit_operand(src, dst, 0); 14880 } 14881 14882 void Assembler::eandq(Register dst, Address src1, Register src2, bool no_flags) { 14883 InstructionMark im(this); 14884 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */); 14885 } 14886 14887 void Assembler::andnq(Register dst, Register src1, Register src2) { 14888 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14889 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14890 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 14891 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 14892 } 14893 14894 void Assembler::andnq(Register dst, Register src1, Address src2) { 14895 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14896 InstructionMark im(this); 14897 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14898 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14899 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 14900 emit_int8((unsigned char)0xF2); 14901 emit_operand(dst, src2, 0); 14902 } 14903 14904 void Assembler::bsfq(Register dst, Register src) { 14905 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 14906 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode); 14907 } 14908 14909 void Assembler::bsrq(Register dst, Register src) { 14910 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 14911 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode); 14912 } 14913 14914 void Assembler::bswapq(Register reg) { 14915 int encode = prefixq_and_encode(reg->encoding(), true /* is_map1 */); 14916 emit_opcode_prefix_and_encoding((unsigned char)0xC8, encode); 14917 } 14918 14919 void Assembler::blsiq(Register dst, Register src) { 14920 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14921 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14922 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 14923 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 14924 } 14925 14926 void Assembler::blsiq(Register dst, Address src) { 14927 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14928 InstructionMark im(this); 14929 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14930 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14931 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 14932 emit_int8((unsigned char)0xF3); 14933 emit_operand(rbx, src, 0); 14934 } 14935 14936 void Assembler::blsmskq(Register dst, Register src) { 14937 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14938 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14939 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 14940 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 14941 } 14942 14943 void Assembler::blsmskq(Register dst, Address src) { 14944 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14945 InstructionMark im(this); 14946 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14947 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14948 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 14949 emit_int8((unsigned char)0xF3); 14950 emit_operand(rdx, src, 0); 14951 } 14952 14953 void Assembler::blsrq(Register dst, Register src) { 14954 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14955 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14956 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true); 14957 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 14958 } 14959 14960 void Assembler::blsrq(Register dst, Address src) { 14961 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 14962 InstructionMark im(this); 14963 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 14964 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 14965 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 14966 emit_int8((unsigned char)0xF3); 14967 emit_operand(rcx, src, 0); 14968 } 14969 14970 void Assembler::cdqq() { 14971 emit_int16(REX_W, (unsigned char)0x99); 14972 } 14973 14974 void Assembler::cdqe() { 14975 emit_int16(REX_W, (unsigned char)0x98); 14976 } 14977 14978 void Assembler::clflush(Address adr) { 14979 assert(VM_Version::supports_clflush(), "should do"); 14980 prefix(adr, true /* is_map1 */); 14981 emit_int8((unsigned char)0xAE); 14982 emit_operand(rdi, adr, 0); 14983 } 14984 14985 void Assembler::clflushopt(Address adr) { 14986 assert(VM_Version::supports_clflushopt(), "should do!"); 14987 // adr should be base reg only with no index or offset 14988 assert(adr.index() == noreg, "index should be noreg"); 14989 assert(adr.scale() == Address::no_scale, "scale should be no_scale"); 14990 assert(adr.disp() == 0, "displacement should be 0"); 14991 // instruction prefix is 0x66 14992 emit_int8(0x66); 14993 prefix(adr, true /* is_map1 */); 14994 // opcode family is 0x0F 0xAE 14995 emit_int8((unsigned char)0xAE); 14996 // extended opcode byte is 7 == rdi 14997 emit_operand(rdi, adr, 0); 14998 } 14999 15000 void Assembler::clwb(Address adr) { 15001 assert(VM_Version::supports_clwb(), "should do!"); 15002 // adr should be base reg only with no index or offset 15003 assert(adr.index() == noreg, "index should be noreg"); 15004 assert(adr.scale() == Address::no_scale, "scale should be no_scale"); 15005 assert(adr.disp() == 0, "displacement should be 0"); 15006 // instruction prefix is 0x66 15007 emit_int8(0x66); 15008 prefix(adr, true /* is_map1 */); 15009 // opcode family is 0x0f 0xAE 15010 emit_int8((unsigned char)0xAE); 15011 // extended opcode byte is 6 == rsi 15012 emit_operand(rsi, adr, 0); 15013 } 15014 15015 void Assembler::cmovq(Condition cc, Register dst, Register src) { 15016 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15017 emit_opcode_prefix_and_encoding((0x40 | cc), 0xC0, encode); 15018 } 15019 15020 void Assembler::ecmovq(Condition cc, Register dst, Register src1, Register src2) { 15021 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x40 | cc, false /* no_flags */, true /* is_map1 */, true /* swap */); 15022 } 15023 15024 void Assembler::cmovq(Condition cc, Register dst, Address src) { 15025 InstructionMark im(this); 15026 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15027 emit_prefix_and_int8(prefix, (0x40 | cc)); 15028 emit_operand(dst, src, 0); 15029 } 15030 15031 void Assembler::ecmovq(Condition cc, Register dst, Register src1, Address src2) { 15032 InstructionMark im(this); 15033 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, (0x40 | cc) , false /* no_flags */, true /* is_map1 */); 15034 } 15035 15036 void Assembler::cmpq(Address dst, int32_t imm32) { 15037 InstructionMark im(this); 15038 prefixq(dst); 15039 emit_arith_operand(0x81, as_Register(7), dst, imm32); 15040 } 15041 15042 void Assembler::cmpq(Register dst, int32_t imm32) { 15043 (void) prefixq_and_encode(dst->encoding()); 15044 emit_arith(0x81, 0xF8, dst, imm32); 15045 } 15046 15047 void Assembler::cmpq(Address dst, Register src) { 15048 InstructionMark im(this); 15049 emit_prefix_and_int8(get_prefixq(dst, src), 0x39); 15050 emit_operand(src, dst, 0); 15051 } 15052 15053 void Assembler::cmpq(Register dst, Register src) { 15054 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 15055 emit_arith(0x3B, 0xC0, dst, src); 15056 } 15057 15058 void Assembler::cmpq(Register dst, Address src) { 15059 InstructionMark im(this); 15060 emit_prefix_and_int8(get_prefixq(src, dst), 0x3B); 15061 emit_operand(dst, src, 0); 15062 } 15063 15064 void Assembler::cmpxchgq(Register reg, Address adr) { 15065 InstructionMark im(this); 15066 int prefix = get_prefixq(adr, reg, true /* is_map1 */); 15067 emit_prefix_and_int8(prefix, (unsigned char)0xB1); 15068 emit_operand(reg, adr, 0); 15069 } 15070 15071 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) { 15072 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15073 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true); 15074 emit_int16(0x2A, (0xC0 | encode)); 15075 } 15076 15077 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) { 15078 InstructionMark im(this); 15079 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15080 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 15081 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 15082 emit_int8(0x2A); 15083 emit_operand(dst, src, 0); 15084 } 15085 15086 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) { 15087 InstructionMark im(this); 15088 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15089 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 15090 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 15091 emit_int8(0x2A); 15092 emit_operand(dst, src, 0); 15093 } 15094 15095 void Assembler::cvttsd2siq(Register dst, Address src) { 15096 // F2 REX.W 0F 2C /r 15097 // CVTTSD2SI r64, xmm1/m64 15098 InstructionMark im(this); 15099 emit_int8((unsigned char)0xF2); 15100 prefixq(src, dst, true /* is_map1 */); 15101 emit_int8((unsigned char)0x2C); 15102 emit_operand(dst, src, 0); 15103 } 15104 15105 void Assembler::evcvttsd2sisl(Register dst, XMMRegister src) { 15106 assert(VM_Version::supports_avx10_2(), ""); 15107 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15108 attributes.set_is_evex_instruction(); 15109 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes); 15110 emit_int16(0x6D, (0xC0 | encode)); 15111 } 15112 15113 void Assembler::evcvttsd2sisl(Register dst, Address src) { 15114 assert(VM_Version::supports_avx10_2(), ""); 15115 InstructionMark im(this); 15116 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15117 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 15118 attributes.set_is_evex_instruction(); 15119 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes); 15120 emit_int8((unsigned char)0x6D); 15121 emit_operand(dst, src, 0); 15122 } 15123 15124 void Assembler::evcvttsd2sisq(Register dst, XMMRegister src) { 15125 assert(VM_Version::supports_avx10_2(), ""); 15126 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15127 attributes.set_is_evex_instruction(); 15128 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes); 15129 emit_int16(0x6D, (0xC0 | encode)); 15130 } 15131 15132 void Assembler::evcvttsd2sisq(Register dst, Address src) { 15133 assert(VM_Version::supports_avx10_2(), ""); 15134 InstructionMark im(this); 15135 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15136 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 15137 attributes.set_is_evex_instruction(); 15138 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes); 15139 emit_int8((unsigned char)0x6D); 15140 emit_operand(dst, src, 0); 15141 } 15142 15143 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 15144 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15145 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 15146 emit_int16(0x2C, (0xC0 | encode)); 15147 } 15148 15149 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 15150 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15151 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 15152 emit_int16(0x2D, (0xC0 | encode)); 15153 } 15154 15155 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 15156 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15157 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 15158 emit_int16(0x2C, (0xC0 | encode)); 15159 } 15160 15161 void Assembler::decl(Register dst) { 15162 // Don't use it directly. Use MacroAssembler::decrementl() instead. 15163 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode) 15164 int encode = prefix_and_encode(dst->encoding()); 15165 emit_int16((unsigned char)0xFF, (0xC8 | encode)); 15166 } 15167 15168 void Assembler::edecl(Register dst, Register src, bool no_flags) { 15169 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15170 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15171 emit_int16((unsigned char)0xFF, (0xC8 | encode)); 15172 } 15173 15174 void Assembler::decq(Register dst) { 15175 // Don't use it directly. Use MacroAssembler::decrementq() instead. 15176 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 15177 int encode = prefixq_and_encode(dst->encoding()); 15178 emit_int16((unsigned char)0xFF, 0xC8 | encode); 15179 } 15180 15181 void Assembler::edecq(Register dst, Register src, bool no_flags) { 15182 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15183 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 15184 emit_int16((unsigned char)0xFF, (0xC8 | encode)); 15185 } 15186 15187 void Assembler::decq(Address dst) { 15188 // Don't use it directly. Use MacroAssembler::decrementq() instead. 15189 InstructionMark im(this); 15190 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xFF); 15191 emit_operand(rcx, dst, 0); 15192 } 15193 15194 void Assembler::edecq(Register dst, Address src, bool no_flags) { 15195 InstructionMark im(this); 15196 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15197 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15198 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15199 emit_int8((unsigned char)0xFF); 15200 emit_operand(rcx, src, 0); 15201 } 15202 15203 // can't use REX2 15204 void Assembler::fxrstor(Address src) { 15205 InstructionMark im(this); 15206 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE); 15207 emit_operand(as_Register(1), src, 0); 15208 } 15209 15210 // can't use REX2 15211 void Assembler::xrstor(Address src) { 15212 InstructionMark im(this); 15213 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE); 15214 emit_operand(as_Register(5), src, 0); 15215 } 15216 15217 // can't use REX2 15218 void Assembler::fxsave(Address dst) { 15219 InstructionMark im(this); 15220 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE); 15221 emit_operand(as_Register(0), dst, 0); 15222 } 15223 15224 // cant use REX2 15225 void Assembler::xsave(Address dst) { 15226 InstructionMark im(this); 15227 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE); 15228 emit_operand(as_Register(4), dst, 0); 15229 } 15230 15231 void Assembler::idivq(Register src) { 15232 int encode = prefixq_and_encode(src->encoding()); 15233 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 15234 } 15235 15236 void Assembler::eidivq(Register src, bool no_flags) { 15237 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15238 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15239 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 15240 } 15241 15242 void Assembler::divq(Register src) { 15243 int encode = prefixq_and_encode(src->encoding()); 15244 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 15245 } 15246 15247 void Assembler::edivq(Register src, bool no_flags) { 15248 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15249 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15250 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 15251 } 15252 15253 void Assembler::imulq(Register dst, Register src) { 15254 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15255 emit_opcode_prefix_and_encoding((unsigned char)0xAF, 0xC0, encode); 15256 } 15257 15258 void Assembler::eimulq(Register dst, Register src, bool no_flags) { 15259 if (is_demotable(no_flags, dst->encoding(), src->encoding())) { 15260 return imulq(dst); 15261 } 15262 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15263 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15264 emit_int16((unsigned char)0xAF, (0xC0 | encode)); 15265 } 15266 15267 void Assembler::eimulq(Register dst, Register src1, Register src2, bool no_flags) { 15268 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */); 15269 } 15270 15271 void Assembler::imulq(Register src) { 15272 int encode = prefixq_and_encode(src->encoding()); 15273 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 15274 } 15275 15276 void Assembler::eimulq(Register src, bool no_flags) { 15277 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15278 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15279 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 15280 } 15281 15282 void Assembler::imulq(Register dst, Address src, int32_t value) { 15283 InstructionMark im(this); 15284 prefixq(src, dst); 15285 if (is8bit(value)) { 15286 emit_int8((unsigned char)0x6B); 15287 emit_operand(dst, src, 1); 15288 emit_int8(value); 15289 } else { 15290 emit_int8((unsigned char)0x69); 15291 emit_operand(dst, src, 4); 15292 emit_int32(value); 15293 } 15294 } 15295 15296 void Assembler::eimulq(Register dst, Address src, int32_t value, bool no_flags) { 15297 InstructionMark im(this); 15298 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15299 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 15300 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15301 if (is8bit(value)) { 15302 emit_int8((unsigned char)0x6B); 15303 emit_operand(dst, src, 1); 15304 emit_int8(value); 15305 } else { 15306 emit_int8((unsigned char)0x69); 15307 emit_operand(dst, src, 4); 15308 emit_int32(value); 15309 } 15310 } 15311 15312 void Assembler::imulq(Register dst, Register src, int value) { 15313 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 15314 if (is8bit(value)) { 15315 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF)); 15316 } else { 15317 emit_int16(0x69, (0xC0 | encode)); 15318 emit_int32(value); 15319 } 15320 } 15321 15322 void Assembler::eimulq(Register dst, Register src, int value, bool no_flags) { 15323 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15324 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15325 if (is8bit(value)) { 15326 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF)); 15327 } else { 15328 emit_int16(0x69, (0xC0 | encode)); 15329 emit_int32(value); 15330 } 15331 } 15332 15333 void Assembler::imulq(Register dst, Address src) { 15334 InstructionMark im(this); 15335 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15336 emit_prefix_and_int8(prefix, (unsigned char)0xAF); 15337 emit_operand(dst, src, 0); 15338 } 15339 15340 void Assembler::eimulq(Register dst, Address src, bool no_flags) { 15341 InstructionMark im(this); 15342 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15343 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15344 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15345 15346 emit_int8((unsigned char)0xAF); 15347 emit_operand(dst, src, 0); 15348 } 15349 15350 void Assembler::eimulq(Register dst, Register src1, Address src2, bool no_flags) { 15351 InstructionMark im(this); 15352 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, (unsigned char)0xAF, no_flags, true /* is_map1 */); 15353 } 15354 15355 void Assembler::incl(Register dst) { 15356 // Don't use it directly. Use MacroAssembler::incrementl() instead. 15357 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 15358 int encode = prefix_and_encode(dst->encoding()); 15359 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 15360 } 15361 15362 void Assembler::eincl(Register dst, Register src, bool no_flags) { 15363 // Don't use it directly. Use MacroAssembler::incrementl() instead. 15364 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 15365 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15366 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15367 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 15368 } 15369 15370 void Assembler::incq(Register dst) { 15371 // Don't use it directly. Use MacroAssembler::incrementq() instead. 15372 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 15373 int encode = prefixq_and_encode(dst->encoding()); 15374 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 15375 } 15376 15377 void Assembler::eincq(Register dst, Register src, bool no_flags) { 15378 // Don't use it directly. Use MacroAssembler::incrementq() instead. 15379 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 15380 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15381 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 15382 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 15383 } 15384 15385 void Assembler::incq(Address dst) { 15386 // Don't use it directly. Use MacroAssembler::incrementq() instead. 15387 InstructionMark im(this); 15388 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xFF); 15389 emit_operand(rax, dst, 0); 15390 } 15391 15392 void Assembler::eincq(Register dst, Address src, bool no_flags) { 15393 // Don't use it directly. Use MacroAssembler::incrementq() instead. 15394 InstructionMark im(this); 15395 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15396 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15397 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15398 emit_int8((unsigned char) 0xFF); 15399 emit_operand(rax, src, 0); 15400 } 15401 15402 void Assembler::lea(Register dst, Address src) { 15403 leaq(dst, src); 15404 } 15405 15406 void Assembler::leaq(Register dst, Address src) { 15407 InstructionMark im(this); 15408 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x8D); 15409 emit_operand(dst, src, 0); 15410 } 15411 15412 void Assembler::mov64(Register dst, int64_t imm64) { 15413 InstructionMark im(this); 15414 int encode = prefixq_and_encode(dst->encoding()); 15415 emit_int8(0xB8 | encode); 15416 emit_int64(imm64); 15417 } 15418 15419 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) { 15420 InstructionMark im(this); 15421 int encode = prefixq_and_encode(dst->encoding()); 15422 emit_int8(0xB8 | encode); 15423 emit_data64(imm64, rtype, format); 15424 } 15425 15426 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) { 15427 InstructionMark im(this); 15428 int encode = prefixq_and_encode(dst->encoding()); 15429 emit_int8(0xB8 | encode); 15430 emit_data64(imm64, rspec); 15431 } 15432 15433 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) { 15434 InstructionMark im(this); 15435 int encode = prefix_and_encode(dst->encoding()); 15436 emit_int8(0xB8 | encode); 15437 emit_data((int)imm32, rspec, narrow_oop_operand); 15438 } 15439 15440 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) { 15441 InstructionMark im(this); 15442 prefix(dst); 15443 emit_int8((unsigned char)0xC7); 15444 emit_operand(rax, dst, 4); 15445 emit_data((int)imm32, rspec, narrow_oop_operand); 15446 } 15447 15448 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) { 15449 InstructionMark im(this); 15450 int encode = prefix_and_encode(src1->encoding()); 15451 emit_int16((unsigned char)0x81, (0xF8 | encode)); 15452 emit_data((int)imm32, rspec, narrow_oop_operand); 15453 } 15454 15455 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) { 15456 InstructionMark im(this); 15457 prefix(src1); 15458 emit_int8((unsigned char)0x81); 15459 emit_operand(rax, src1, 4); 15460 emit_data((int)imm32, rspec, narrow_oop_operand); 15461 } 15462 15463 void Assembler::lzcntq(Register dst, Register src) { 15464 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 15465 emit_int8((unsigned char)0xF3); 15466 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15467 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode); 15468 } 15469 15470 void Assembler::elzcntq(Register dst, Register src, bool no_flags) { 15471 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 15472 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15473 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15474 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 15475 } 15476 15477 void Assembler::lzcntq(Register dst, Address src) { 15478 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 15479 InstructionMark im(this); 15480 emit_int8((unsigned char)0xF3); 15481 prefixq(src, dst, true /* is_map1 */); 15482 emit_int8((unsigned char)0xBD); 15483 emit_operand(dst, src, 0); 15484 } 15485 15486 void Assembler::elzcntq(Register dst, Address src, bool no_flags) { 15487 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 15488 InstructionMark im(this); 15489 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15490 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15491 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15492 emit_int8((unsigned char)0xF5); 15493 emit_operand(dst, src, 0); 15494 } 15495 15496 void Assembler::movdq(XMMRegister dst, Register src) { 15497 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15498 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 15499 emit_int16(0x6E, (0xC0 | encode)); 15500 } 15501 15502 void Assembler::movdq(Register dst, XMMRegister src) { 15503 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15504 // swap src/dst to get correct prefix 15505 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true); 15506 emit_int16(0x7E, 15507 (0xC0 | encode)); 15508 } 15509 15510 void Assembler::movq(Register dst, Register src) { 15511 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 15512 emit_int16((unsigned char)0x8B, 15513 (0xC0 | encode)); 15514 } 15515 15516 void Assembler::movq(Register dst, Address src) { 15517 InstructionMark im(this); 15518 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x8B); 15519 emit_operand(dst, src, 0); 15520 } 15521 15522 void Assembler::movq(Address dst, Register src) { 15523 InstructionMark im(this); 15524 emit_prefix_and_int8(get_prefixq(dst, src), (unsigned char)0x89); 15525 emit_operand(src, dst, 0); 15526 } 15527 15528 void Assembler::movq(Address dst, int32_t imm32) { 15529 InstructionMark im(this); 15530 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC7); 15531 emit_operand(as_Register(0), dst, 4); 15532 emit_int32(imm32); 15533 } 15534 15535 void Assembler::movq(Register dst, int32_t imm32) { 15536 int encode = prefixq_and_encode(dst->encoding()); 15537 emit_int16((unsigned char)0xC7, (0xC0 | encode)); 15538 emit_int32(imm32); 15539 } 15540 15541 void Assembler::movsbq(Register dst, Address src) { 15542 InstructionMark im(this); 15543 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15544 emit_prefix_and_int8(prefix, (unsigned char)0xBE); 15545 emit_operand(dst, src, 0); 15546 } 15547 15548 void Assembler::movsbq(Register dst, Register src) { 15549 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15550 emit_opcode_prefix_and_encoding((unsigned char)0xBE, 0xC0, encode); 15551 } 15552 15553 void Assembler::movslq(Address dst, int32_t imm32) { 15554 assert(is_simm32(imm32), "lost bits"); 15555 InstructionMark im(this); 15556 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC7); 15557 emit_operand(rax, dst, 4); 15558 emit_int32(imm32); 15559 } 15560 15561 void Assembler::movslq(Register dst, Address src) { 15562 InstructionMark im(this); 15563 emit_prefix_and_int8(get_prefixq(src, dst), 0x63); 15564 emit_operand(dst, src, 0); 15565 } 15566 15567 void Assembler::movslq(Register dst, Register src) { 15568 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 15569 emit_int16(0x63, (0xC0 | encode)); 15570 } 15571 15572 void Assembler::movswq(Register dst, Address src) { 15573 InstructionMark im(this); 15574 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15575 emit_prefix_and_int8(prefix, (unsigned char)0xBF); 15576 emit_operand(dst, src, 0); 15577 } 15578 15579 void Assembler::movswq(Register dst, Register src) { 15580 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15581 emit_opcode_prefix_and_encoding((unsigned char)0xBF, 0xC0, encode); 15582 } 15583 15584 void Assembler::movzbq(Register dst, Address src) { 15585 InstructionMark im(this); 15586 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15587 emit_prefix_and_int8(prefix, (unsigned char)0xB6); 15588 emit_operand(dst, src, 0); 15589 } 15590 15591 void Assembler::movzbq(Register dst, Register src) { 15592 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15593 emit_opcode_prefix_and_encoding((unsigned char)0xB6, 0xC0, encode); 15594 } 15595 15596 void Assembler::movzwq(Register dst, Address src) { 15597 InstructionMark im(this); 15598 int prefix = get_prefixq(src, dst, true /* is_map1 */); 15599 emit_prefix_and_int8(prefix, (unsigned char)0xB7); 15600 emit_operand(dst, src, 0); 15601 } 15602 15603 void Assembler::movzwq(Register dst, Register src) { 15604 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15605 emit_opcode_prefix_and_encoding((unsigned char)0xB7, 0xC0, encode); 15606 } 15607 15608 void Assembler::mulq(Address src) { 15609 InstructionMark im(this); 15610 emit_prefix_and_int8(get_prefixq(src), (unsigned char)0xF7); 15611 emit_operand(rsp, src, 0); 15612 } 15613 15614 void Assembler::emulq(Address src, bool no_flags) { 15615 InstructionMark im(this); 15616 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15617 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15618 eevex_prefix_nf(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15619 emit_int8(0xF7); 15620 emit_operand(rsp, src, 0); 15621 } 15622 15623 void Assembler::mulq(Register src) { 15624 int encode = prefixq_and_encode(src->encoding()); 15625 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 15626 } 15627 15628 void Assembler::emulq(Register src, bool no_flags) { 15629 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15630 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15631 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 15632 } 15633 15634 void Assembler::mulxq(Register dst1, Register dst2, Register src) { 15635 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 15636 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15637 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true); 15638 emit_int16((unsigned char)0xF6, (0xC0 | encode)); 15639 } 15640 15641 void Assembler::negq(Register dst) { 15642 int encode = prefixq_and_encode(dst->encoding()); 15643 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 15644 } 15645 15646 void Assembler::enegq(Register dst, Register src, bool no_flags) { 15647 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15648 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 15649 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 15650 } 15651 15652 void Assembler::negq(Address dst) { 15653 InstructionMark im(this); 15654 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xF7); 15655 emit_operand(as_Register(3), dst, 0); 15656 } 15657 15658 void Assembler::enegq(Register dst, Address src, bool no_flags) { 15659 InstructionMark im(this); 15660 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15661 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15662 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15663 emit_int8((unsigned char)0xF7); 15664 emit_operand(as_Register(3), src, 0); 15665 } 15666 15667 void Assembler::notq(Register dst) { 15668 int encode = prefixq_and_encode(dst->encoding()); 15669 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 15670 } 15671 15672 void Assembler::enotq(Register dst, Register src) { 15673 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15674 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */); 15675 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 15676 } 15677 15678 void Assembler::btq(Register dst, Register src) { 15679 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */); 15680 emit_opcode_prefix_and_encoding((unsigned char)0xA3, 0xC0, encode); 15681 } 15682 15683 void Assembler::btq(Register src, int imm8) { 15684 assert(isByte(imm8), "not a byte"); 15685 int encode = prefixq_and_encode(src->encoding(), true /* is_map1 */); 15686 emit_opcode_prefix_and_encoding((unsigned char)0xBA, 0xE0, encode); 15687 emit_int8(imm8); 15688 } 15689 15690 void Assembler::btsq(Address dst, int imm8) { 15691 assert(isByte(imm8), "not a byte"); 15692 InstructionMark im(this); 15693 int prefix = get_prefixq(dst, true /* is_map1 */); 15694 emit_prefix_and_int8(prefix, (unsigned char)0xBA); 15695 emit_operand(rbp /* 5 */, dst, 1); 15696 emit_int8(imm8); 15697 } 15698 15699 void Assembler::btrq(Address dst, int imm8) { 15700 assert(isByte(imm8), "not a byte"); 15701 InstructionMark im(this); 15702 int prefix = get_prefixq(dst, true /* is_map1 */); 15703 emit_prefix_and_int8(prefix, (unsigned char)0xBA); 15704 emit_operand(rsi /* 6 */, dst, 1); 15705 emit_int8(imm8); 15706 } 15707 15708 void Assembler::orq(Address dst, int32_t imm32) { 15709 InstructionMark im(this); 15710 prefixq(dst); 15711 emit_arith_operand(0x81, as_Register(1), dst, imm32); 15712 } 15713 15714 void Assembler::eorq(Register dst, Address src, int32_t imm32, bool no_flags) { 15715 InstructionMark im(this); 15716 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15717 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15718 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15719 emit_arith_operand(0x81, as_Register(1), src, imm32); 15720 } 15721 15722 void Assembler::orq(Address dst, Register src) { 15723 InstructionMark im(this); 15724 emit_prefix_and_int8(get_prefixq(dst, src), (unsigned char)0x09); 15725 emit_operand(src, dst, 0); 15726 } 15727 15728 void Assembler::eorq(Register dst, Address src1, Register src2, bool no_flags) { 15729 InstructionMark im(this); 15730 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */); 15731 } 15732 15733 void Assembler::orq(Register dst, int32_t imm32) { 15734 (void) prefixq_and_encode(dst->encoding()); 15735 emit_arith(0x81, 0xC8, dst, imm32); 15736 } 15737 15738 void Assembler::eorq(Register dst, Register src, int32_t imm32, bool no_flags) { 15739 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xC8, no_flags); 15740 } 15741 15742 void Assembler::orq_imm32(Register dst, int32_t imm32) { 15743 (void) prefixq_and_encode(dst->encoding()); 15744 emit_arith_imm32(0x81, 0xC8, dst, imm32); 15745 } 15746 15747 void Assembler::eorq_imm32(Register dst, Register src, int32_t imm32, bool no_flags) { 15748 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15749 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 15750 emit_arith_imm32(0x81, 0xC8, src, imm32); 15751 } 15752 15753 void Assembler::orq(Register dst, Address src) { 15754 InstructionMark im(this); 15755 emit_prefix_and_int8(get_prefixq(src, dst), 0x0B); 15756 emit_operand(dst, src, 0); 15757 } 15758 15759 void Assembler::eorq(Register dst, Register src1, Address src2, bool no_flags) { 15760 InstructionMark im(this); 15761 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, no_flags); 15762 } 15763 15764 void Assembler::orq(Register dst, Register src) { 15765 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 15766 emit_arith(0x0B, 0xC0, dst, src); 15767 } 15768 15769 void Assembler::eorq(Register dst, Register src1, Register src2, bool no_flags) { 15770 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, 0xC0, no_flags, true /* is_commutative */); 15771 } 15772 void Assembler::popcntq(Register dst, Address src) { 15773 assert(VM_Version::supports_popcnt(), "must support"); 15774 InstructionMark im(this); 15775 emit_int8((unsigned char)0xF3); 15776 emit_prefix_and_int8(get_prefixq(src, dst, true /* is_map1 */), (unsigned char) 0xB8); 15777 emit_operand(dst, src, 0); 15778 } 15779 15780 void Assembler::epopcntq(Register dst, Address src, bool no_flags) { 15781 assert(VM_Version::supports_popcnt(), "must support"); 15782 InstructionMark im(this); 15783 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15784 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 15785 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15786 emit_int8((unsigned char) 0x88); 15787 emit_operand(dst, src, 0); 15788 } 15789 15790 void Assembler::popcntq(Register dst, Register src) { 15791 assert(VM_Version::supports_popcnt(), "must support"); 15792 emit_int8((unsigned char)0xF3); 15793 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */); 15794 emit_opcode_prefix_and_encoding((unsigned char)0xB8, 0xC0, encode); 15795 } 15796 15797 void Assembler::epopcntq(Register dst, Register src, bool no_flags) { 15798 assert(VM_Version::supports_popcnt(), "must support"); 15799 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 15800 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 15801 emit_int16((unsigned char)0x88, (0xC0 | encode)); 15802 } 15803 15804 void Assembler::popq(Address dst) { 15805 InstructionMark im(this); 15806 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0x8F); 15807 emit_operand(rax, dst, 0); 15808 } 15809 15810 void Assembler::popq(Register dst) { 15811 int encode = prefix_and_encode(dst->encoding()); 15812 emit_int8((unsigned char)0x58 | encode); 15813 } 15814 15815 // Precomputable: popa, pusha, vzeroupper 15816 15817 // The result of these routines are invariant from one invocation to another 15818 // invocation for the duration of a run. Caching the result on bootstrap 15819 // and copying it out on subsequent invocations can thus be beneficial 15820 static bool precomputed = false; 15821 15822 static u_char* popa_code = nullptr; 15823 static int popa_len = 0; 15824 15825 static u_char* pusha_code = nullptr; 15826 static int pusha_len = 0; 15827 15828 static u_char* vzup_code = nullptr; 15829 static int vzup_len = 0; 15830 15831 void Assembler::precompute_instructions() { 15832 assert(!Universe::is_fully_initialized(), "must still be single threaded"); 15833 guarantee(!precomputed, "only once"); 15834 precomputed = true; 15835 ResourceMark rm; 15836 15837 // Make a temporary buffer big enough for the routines we're capturing 15838 int size = UseAPX ? 512 : 256; 15839 char* tmp_code = NEW_RESOURCE_ARRAY(char, size); 15840 CodeBuffer buffer((address)tmp_code, size); 15841 MacroAssembler masm(&buffer); 15842 15843 address begin_popa = masm.code_section()->end(); 15844 masm.popa_uncached(); 15845 address end_popa = masm.code_section()->end(); 15846 masm.pusha_uncached(); 15847 address end_pusha = masm.code_section()->end(); 15848 masm.vzeroupper_uncached(); 15849 address end_vzup = masm.code_section()->end(); 15850 15851 // Save the instructions to permanent buffers. 15852 popa_len = (int)(end_popa - begin_popa); 15853 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal); 15854 memcpy(popa_code, begin_popa, popa_len); 15855 15856 pusha_len = (int)(end_pusha - end_popa); 15857 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal); 15858 memcpy(pusha_code, end_popa, pusha_len); 15859 15860 vzup_len = (int)(end_vzup - end_pusha); 15861 if (vzup_len > 0) { 15862 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal); 15863 memcpy(vzup_code, end_pusha, vzup_len); 15864 } else { 15865 vzup_code = pusha_code; // dummy 15866 } 15867 15868 assert(masm.code()->total_oop_size() == 0 && 15869 masm.code()->total_metadata_size() == 0 && 15870 masm.code()->total_relocation_size() == 0, 15871 "pre-computed code can't reference oops, metadata or contain relocations"); 15872 } 15873 15874 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) { 15875 assert(src != nullptr, "code to copy must have been pre-computed"); 15876 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough"); 15877 address end = code_section->end(); 15878 memcpy(end, src, src_len); 15879 code_section->set_end(end + src_len); 15880 } 15881 15882 15883 // Does not actually store the value of rsp on the stack. 15884 // The slot for rsp just contains an arbitrary value. 15885 void Assembler::pusha() { // 64bit 15886 emit_copy(code_section(), pusha_code, pusha_len); 15887 } 15888 15889 // Does not actually store the value of rsp on the stack. 15890 // The slot for rsp just contains an arbitrary value. 15891 void Assembler::pusha_uncached() { // 64bit 15892 if (UseAPX) { 15893 // Data being pushed by PUSH2 must be 16B-aligned on the stack, for this push rax upfront 15894 // and use it as a temporary register for stack alignment. 15895 pushp(rax); 15896 // Move original stack pointer to RAX and align stack pointer to 16B boundary. 15897 movq(rax, rsp); 15898 andq(rsp, -(StackAlignmentInBytes)); 15899 // Push pair of original stack pointer along with remaining registers 15900 // at 16B aligned boundary. 15901 push2p(rax, r31); 15902 // Restore the original contents of RAX register. 15903 movq(rax, Address(rax)); 15904 push2p(r30, r29); 15905 push2p(r28, r27); 15906 push2p(r26, r25); 15907 push2p(r24, r23); 15908 push2p(r22, r21); 15909 push2p(r20, r19); 15910 push2p(r18, r17); 15911 push2p(r16, r15); 15912 push2p(r14, r13); 15913 push2p(r12, r11); 15914 push2p(r10, r9); 15915 push2p(r8, rdi); 15916 push2p(rsi, rbp); 15917 push2p(rbx, rdx); 15918 // To maintain 16 byte alignment after rcx is pushed. 15919 subq(rsp, 8); 15920 pushp(rcx); 15921 } else { 15922 subq(rsp, 16 * wordSize); 15923 movq(Address(rsp, 15 * wordSize), rax); 15924 movq(Address(rsp, 14 * wordSize), rcx); 15925 movq(Address(rsp, 13 * wordSize), rdx); 15926 movq(Address(rsp, 12 * wordSize), rbx); 15927 // Skip rsp as the value is normally not used. There are a few places where 15928 // the original value of rsp needs to be known but that can be computed 15929 // from the value of rsp immediately after pusha (rsp + 16 * wordSize). 15930 // FIXME: For APX any such direct access should also consider EGPR size 15931 // during address compution. 15932 movq(Address(rsp, 10 * wordSize), rbp); 15933 movq(Address(rsp, 9 * wordSize), rsi); 15934 movq(Address(rsp, 8 * wordSize), rdi); 15935 movq(Address(rsp, 7 * wordSize), r8); 15936 movq(Address(rsp, 6 * wordSize), r9); 15937 movq(Address(rsp, 5 * wordSize), r10); 15938 movq(Address(rsp, 4 * wordSize), r11); 15939 movq(Address(rsp, 3 * wordSize), r12); 15940 movq(Address(rsp, 2 * wordSize), r13); 15941 movq(Address(rsp, wordSize), r14); 15942 movq(Address(rsp, 0), r15); 15943 } 15944 } 15945 15946 void Assembler::popa() { // 64bit 15947 emit_copy(code_section(), popa_code, popa_len); 15948 } 15949 15950 void Assembler::popa_uncached() { // 64bit 15951 if (UseAPX) { 15952 popp(rcx); 15953 addq(rsp, 8); 15954 // Data being popped by POP2 must be 16B-aligned on the stack. 15955 pop2p(rdx, rbx); 15956 pop2p(rbp, rsi); 15957 pop2p(rdi, r8); 15958 pop2p(r9, r10); 15959 pop2p(r11, r12); 15960 pop2p(r13, r14); 15961 pop2p(r15, r16); 15962 pop2p(r17, r18); 15963 pop2p(r19, r20); 15964 pop2p(r21, r22); 15965 pop2p(r23, r24); 15966 pop2p(r25, r26); 15967 pop2p(r27, r28); 15968 pop2p(r29, r30); 15969 // Popped value in RAX holds original unaligned stack pointer. 15970 pop2p(r31, rax); 15971 // Reinstantiate original stack pointer. 15972 movq(rsp, rax); 15973 popp(rax); 15974 } else { 15975 movq(r15, Address(rsp, 0)); 15976 movq(r14, Address(rsp, wordSize)); 15977 movq(r13, Address(rsp, 2 * wordSize)); 15978 movq(r12, Address(rsp, 3 * wordSize)); 15979 movq(r11, Address(rsp, 4 * wordSize)); 15980 movq(r10, Address(rsp, 5 * wordSize)); 15981 movq(r9, Address(rsp, 6 * wordSize)); 15982 movq(r8, Address(rsp, 7 * wordSize)); 15983 movq(rdi, Address(rsp, 8 * wordSize)); 15984 movq(rsi, Address(rsp, 9 * wordSize)); 15985 movq(rbp, Address(rsp, 10 * wordSize)); 15986 // Skip rsp as it is restored automatically to the value 15987 // before the corresponding pusha when popa is done. 15988 movq(rbx, Address(rsp, 12 * wordSize)); 15989 movq(rdx, Address(rsp, 13 * wordSize)); 15990 movq(rcx, Address(rsp, 14 * wordSize)); 15991 movq(rax, Address(rsp, 15 * wordSize)); 15992 15993 addq(rsp, 16 * wordSize); 15994 } 15995 } 15996 15997 void Assembler::vzeroupper() { 15998 emit_copy(code_section(), vzup_code, vzup_len); 15999 } 16000 16001 void Assembler::vzeroall() { 16002 assert(VM_Version::supports_avx(), "requires AVX"); 16003 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 16004 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 16005 emit_int8(0x77); 16006 } 16007 16008 void Assembler::pushq(Address src) { 16009 InstructionMark im(this); 16010 emit_prefix_and_int8(get_prefixq(src), (unsigned char)0xFF); 16011 emit_operand(rsi, src, 0); 16012 } 16013 16014 void Assembler::rclq(Register dst, int imm8) { 16015 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16016 int encode = prefixq_and_encode(dst->encoding()); 16017 if (imm8 == 1) { 16018 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 16019 } else { 16020 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 16021 } 16022 } 16023 16024 void Assembler::erclq(Register dst, Register src, int imm8) { 16025 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16026 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16027 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */); 16028 if (imm8 == 1) { 16029 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 16030 } else { 16031 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 16032 } 16033 } 16034 16035 void Assembler::rcrq(Register dst, int imm8) { 16036 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16037 int encode = prefixq_and_encode(dst->encoding()); 16038 if (imm8 == 1) { 16039 emit_int16((unsigned char)0xD1, (0xD8 | encode)); 16040 } else { 16041 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8); 16042 } 16043 } 16044 16045 void Assembler::ercrq(Register dst, Register src, int imm8) { 16046 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16047 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16048 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */); 16049 if (imm8 == 1) { 16050 emit_int16((unsigned char)0xD1, (0xD8 | encode)); 16051 } else { 16052 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8); 16053 } 16054 } 16055 16056 void Assembler::rorxl(Register dst, Register src, int imm8) { 16057 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 16058 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16059 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes, true); 16060 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8); 16061 } 16062 16063 void Assembler::rorxl(Register dst, Address src, int imm8) { 16064 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 16065 InstructionMark im(this); 16066 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16067 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit); 16068 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 16069 emit_int8((unsigned char)0xF0); 16070 emit_operand(dst, src, 1); 16071 emit_int8(imm8); 16072 } 16073 16074 void Assembler::rorxq(Register dst, Register src, int imm8) { 16075 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 16076 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16077 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes, true); 16078 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8); 16079 } 16080 16081 void Assembler::rorxq(Register dst, Address src, int imm8) { 16082 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 16083 InstructionMark im(this); 16084 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16085 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16086 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 16087 emit_int8((unsigned char)0xF0); 16088 emit_operand(dst, src, 1); 16089 emit_int8(imm8); 16090 } 16091 16092 void Assembler::salq(Address dst, int imm8) { 16093 InstructionMark im(this); 16094 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16095 if (imm8 == 1) { 16096 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1); 16097 emit_operand(as_Register(4), dst, 0); 16098 } 16099 else { 16100 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1); 16101 emit_operand(as_Register(4), dst, 1); 16102 emit_int8(imm8); 16103 } 16104 } 16105 16106 void Assembler::esalq(Register dst, Address src, int imm8, bool no_flags) { 16107 InstructionMark im(this); 16108 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16109 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16110 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16111 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16112 if (imm8 == 1) { 16113 emit_int8((unsigned char)0xD1); 16114 emit_operand(as_Register(4), src, 0); 16115 } 16116 else { 16117 emit_int8((unsigned char)0xC1); 16118 emit_operand(as_Register(4), src, 1); 16119 emit_int8(imm8); 16120 } 16121 } 16122 16123 void Assembler::salq(Address dst) { 16124 InstructionMark im(this); 16125 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3); 16126 emit_operand(as_Register(4), dst, 0); 16127 } 16128 16129 void Assembler::esalq(Register dst, Address src, bool no_flags) { 16130 InstructionMark im(this); 16131 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16132 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16133 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16134 emit_int8((unsigned char)0xD3); 16135 emit_operand(as_Register(4), src, 0); 16136 } 16137 16138 void Assembler::salq(Register dst, int imm8) { 16139 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16140 int encode = prefixq_and_encode(dst->encoding()); 16141 if (imm8 == 1) { 16142 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 16143 } else { 16144 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 16145 } 16146 } 16147 16148 void Assembler::esalq(Register dst, Register src, int imm8, bool no_flags) { 16149 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16150 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16151 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16152 if (imm8 == 1) { 16153 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 16154 } else { 16155 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 16156 } 16157 } 16158 16159 void Assembler::salq(Register dst) { 16160 int encode = prefixq_and_encode(dst->encoding()); 16161 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 16162 } 16163 16164 void Assembler::esalq(Register dst, Register src, bool no_flags) { 16165 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16166 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16167 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 16168 } 16169 16170 void Assembler::sarq(Address dst, int imm8) { 16171 InstructionMark im(this); 16172 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16173 if (imm8 == 1) { 16174 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1); 16175 emit_operand(as_Register(7), dst, 0); 16176 } 16177 else { 16178 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1); 16179 emit_operand(as_Register(7), dst, 1); 16180 emit_int8(imm8); 16181 } 16182 } 16183 16184 void Assembler::esarq(Register dst, Address src, int imm8, bool no_flags) { 16185 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16186 InstructionMark im(this); 16187 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16188 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16189 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16190 if (imm8 == 1) { 16191 emit_int8((unsigned char)0xD1); 16192 emit_operand(as_Register(7), src, 0); 16193 } 16194 else { 16195 emit_int8((unsigned char)0xC1); 16196 emit_operand(as_Register(7), src, 1); 16197 emit_int8(imm8); 16198 } 16199 } 16200 16201 void Assembler::sarq(Address dst) { 16202 InstructionMark im(this); 16203 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3); 16204 emit_operand(as_Register(7), dst, 0); 16205 } 16206 16207 void Assembler::esarq(Register dst, Address src, bool no_flags) { 16208 InstructionMark im(this); 16209 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16210 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16211 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16212 emit_int8((unsigned char)0xD3); 16213 emit_operand(as_Register(7), src, 0); 16214 } 16215 16216 void Assembler::sarq(Register dst, int imm8) { 16217 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16218 int encode = prefixq_and_encode(dst->encoding()); 16219 if (imm8 == 1) { 16220 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 16221 } else { 16222 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 16223 } 16224 } 16225 16226 void Assembler::esarq(Register dst, Register src, int imm8, bool no_flags) { 16227 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16228 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16229 if (imm8 == 1) { 16230 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 16231 } else { 16232 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 16233 } 16234 } 16235 16236 void Assembler::sarq(Register dst) { 16237 int encode = prefixq_and_encode(dst->encoding()); 16238 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 16239 } 16240 16241 void Assembler::esarq(Register dst, Register src, bool no_flags) { 16242 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16243 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16244 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 16245 } 16246 16247 void Assembler::sbbq(Address dst, int32_t imm32) { 16248 InstructionMark im(this); 16249 prefixq(dst); 16250 emit_arith_operand(0x81, rbx, dst, imm32); 16251 } 16252 16253 void Assembler::sbbq(Register dst, int32_t imm32) { 16254 (void) prefixq_and_encode(dst->encoding()); 16255 emit_arith(0x81, 0xD8, dst, imm32); 16256 } 16257 16258 void Assembler::sbbq(Register dst, Address src) { 16259 InstructionMark im(this); 16260 emit_prefix_and_int8(get_prefixq(src, dst), 0x1B); 16261 emit_operand(dst, src, 0); 16262 } 16263 16264 void Assembler::sbbq(Register dst, Register src) { 16265 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 16266 emit_arith(0x1B, 0xC0, dst, src); 16267 } 16268 16269 void Assembler::shlq(Register dst, int imm8) { 16270 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16271 int encode = prefixq_and_encode(dst->encoding()); 16272 if (imm8 == 1) { 16273 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 16274 } else { 16275 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 16276 } 16277 } 16278 16279 void Assembler::eshlq(Register dst, Register src, int imm8, bool no_flags) { 16280 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16281 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16282 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16283 if (imm8 == 1 ) { 16284 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 16285 } else { 16286 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 16287 } 16288 } 16289 16290 void Assembler::shlq(Register dst) { 16291 int encode = prefixq_and_encode(dst->encoding()); 16292 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 16293 } 16294 16295 void Assembler::eshlq(Register dst, Register src, bool no_flags) { 16296 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16297 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16298 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 16299 } 16300 16301 void Assembler::shrq(Register dst, int imm8) { 16302 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16303 int encode = prefixq_and_encode(dst->encoding()); 16304 if (imm8 == 1) { 16305 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 16306 } 16307 else { 16308 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 16309 } 16310 } 16311 16312 void Assembler::eshrq(Register dst, Register src, int imm8, bool no_flags) { 16313 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16314 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16315 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16316 if (imm8 == 1) { 16317 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 16318 } 16319 else { 16320 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 16321 } 16322 } 16323 16324 void Assembler::shrq(Register dst) { 16325 int encode = prefixq_and_encode(dst->encoding()); 16326 emit_int16((unsigned char)0xD3, 0xE8 | encode); 16327 } 16328 16329 void Assembler::eshrq(Register dst, Register src, bool no_flags) { 16330 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16331 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16332 emit_int16((unsigned char)0xD3, (0xE8 | encode)); 16333 } 16334 16335 void Assembler::shrq(Address dst) { 16336 InstructionMark im(this); 16337 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3); 16338 emit_operand(as_Register(5), dst, 0); 16339 } 16340 16341 void Assembler::eshrq(Register dst, Address src, bool no_flags) { 16342 InstructionMark im(this); 16343 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16344 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16345 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16346 emit_int8((unsigned char)0xD3); 16347 emit_operand(as_Register(5), src, 0); 16348 } 16349 16350 void Assembler::shrq(Address dst, int imm8) { 16351 InstructionMark im(this); 16352 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16353 if (imm8 == 1) { 16354 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1); 16355 emit_operand(as_Register(5), dst, 0); 16356 } 16357 else { 16358 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1); 16359 emit_operand(as_Register(5), dst, 1); 16360 emit_int8(imm8); 16361 } 16362 } 16363 16364 void Assembler::eshrq(Register dst, Address src, int imm8, bool no_flags) { 16365 InstructionMark im(this); 16366 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 16367 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16368 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16369 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16370 if (imm8 == 1) { 16371 emit_int8((unsigned char)0xD1); 16372 emit_operand(as_Register(5), src, 0); 16373 } 16374 else { 16375 emit_int8((unsigned char)0xC1); 16376 emit_operand(as_Register(5), src, 1); 16377 emit_int8(imm8); 16378 } 16379 } 16380 16381 void Assembler::subq(Address dst, int32_t imm32) { 16382 InstructionMark im(this); 16383 prefixq(dst); 16384 emit_arith_operand(0x81, rbp, dst, imm32); 16385 } 16386 16387 void Assembler::esubq(Register dst, Address src, int32_t imm32, bool no_flags) { 16388 InstructionMark im(this); 16389 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16390 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16391 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16392 emit_arith_operand(0x81, rbp, src, imm32); 16393 } 16394 16395 void Assembler::subq(Address dst, Register src) { 16396 InstructionMark im(this); 16397 emit_prefix_and_int8(get_prefixq(dst, src), 0x29); 16398 emit_operand(src, dst, 0); 16399 } 16400 16401 void Assembler::esubq(Register dst, Address src1, Register src2, bool no_flags) { 16402 InstructionMark im(this); 16403 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16404 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16405 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16406 emit_int8(0x29); 16407 emit_operand(src2, src1, 0); 16408 } 16409 16410 void Assembler::subq(Register dst, int32_t imm32) { 16411 (void) prefixq_and_encode(dst->encoding()); 16412 emit_arith(0x81, 0xE8, dst, imm32); 16413 } 16414 16415 void Assembler::esubq(Register dst, Register src, int32_t imm32, bool no_flags) { 16416 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xE8, no_flags); 16417 } 16418 16419 // Force generation of a 4 byte immediate value even if it fits into 8bit 16420 void Assembler::subq_imm32(Register dst, int32_t imm32) { 16421 (void) prefixq_and_encode(dst->encoding()); 16422 emit_arith_imm32(0x81, 0xE8, dst, imm32); 16423 } 16424 16425 void Assembler::esubq_imm32(Register dst, Register src, int32_t imm32, bool no_flags) { 16426 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16427 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16428 emit_arith_imm32(0x81, 0xE8, src, imm32); 16429 } 16430 16431 void Assembler::subq(Register dst, Address src) { 16432 InstructionMark im(this); 16433 emit_prefix_and_int8(get_prefixq(src, dst), 0x2B); 16434 emit_operand(dst, src, 0); 16435 } 16436 16437 void Assembler::esubq(Register dst, Register src1, Address src2, bool no_flags) { 16438 InstructionMark im(this); 16439 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x2B, no_flags); 16440 } 16441 16442 void Assembler::subq(Register dst, Register src) { 16443 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 16444 emit_arith(0x2B, 0xC0, dst, src); 16445 } 16446 16447 void Assembler::esubq(Register dst, Register src1, Register src2, bool no_flags) { 16448 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16449 // NDD shares its encoding bits with NDS bits for regular EVEX instruction. 16450 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine. 16451 (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */); 16452 emit_arith(0x2B, 0xC0, src1, src2); 16453 } 16454 16455 void Assembler::testq(Address dst, int32_t imm32) { 16456 InstructionMark im(this); 16457 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xF7); 16458 emit_operand(as_Register(0), dst, 4); 16459 emit_int32(imm32); 16460 } 16461 16462 void Assembler::testq(Register dst, int32_t imm32) { 16463 // not using emit_arith because test 16464 // doesn't support sign-extension of 16465 // 8bit operands 16466 if (dst == rax) { 16467 prefix(REX_W); 16468 emit_int8((unsigned char)0xA9); 16469 emit_int32(imm32); 16470 } else { 16471 int encode = dst->encoding(); 16472 encode = prefixq_and_encode(encode); 16473 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 16474 emit_int32(imm32); 16475 } 16476 } 16477 16478 void Assembler::testq(Register dst, Register src) { 16479 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 16480 emit_arith(0x85, 0xC0, dst, src); 16481 } 16482 16483 void Assembler::testq(Register dst, Address src) { 16484 InstructionMark im(this); 16485 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x85); 16486 emit_operand(dst, src, 0); 16487 } 16488 16489 void Assembler::xaddq(Address dst, Register src) { 16490 InstructionMark im(this); 16491 int prefix = get_prefixq(dst, src, true /* is_map1 */); 16492 emit_prefix_and_int8(prefix, (unsigned char)0xC1); 16493 emit_operand(src, dst, 0); 16494 } 16495 16496 void Assembler::xchgq(Register dst, Address src) { 16497 InstructionMark im(this); 16498 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x87); 16499 emit_operand(dst, src, 0); 16500 } 16501 16502 void Assembler::xchgq(Register dst, Register src) { 16503 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 16504 emit_int16((unsigned char)0x87, (0xc0 | encode)); 16505 } 16506 16507 void Assembler::xorq(Register dst, Register src) { 16508 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 16509 emit_arith(0x33, 0xC0, dst, src); 16510 } 16511 16512 void Assembler::exorq(Register dst, Register src1, Register src2, bool no_flags) { 16513 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, 0xC0, no_flags, true /* is_commutative */); 16514 } 16515 16516 void Assembler::xorq(Register dst, Address src) { 16517 InstructionMark im(this); 16518 emit_prefix_and_int8(get_prefixq(src, dst), 0x33); 16519 emit_operand(dst, src, 0); 16520 } 16521 16522 void Assembler::exorq(Register dst, Register src1, Address src2, bool no_flags) { 16523 InstructionMark im(this); 16524 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, no_flags); 16525 } 16526 16527 void Assembler::xorq(Register dst, int32_t imm32) { 16528 (void) prefixq_and_encode(dst->encoding()); 16529 emit_arith(0x81, 0xF0, dst, imm32); 16530 } 16531 16532 void Assembler::exorq(Register dst, Register src, int32_t imm32, bool no_flags) { 16533 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xF0, no_flags); 16534 } 16535 16536 void Assembler::xorq(Address dst, int32_t imm32) { 16537 InstructionMark im(this); 16538 prefixq(dst); 16539 emit_arith_operand(0x81, as_Register(6), dst, imm32); 16540 } 16541 16542 void Assembler::exorq(Register dst, Address src, int32_t imm32, bool no_flags) { 16543 InstructionMark im(this); 16544 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16545 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit); 16546 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags); 16547 emit_arith_operand(0x81, as_Register(6), src, imm32); 16548 } 16549 16550 void Assembler::xorq(Address dst, Register src) { 16551 InstructionMark im(this); 16552 emit_prefix_and_int8(get_prefixq(dst, src), 0x31); 16553 emit_operand(src, dst, 0); 16554 } 16555 16556 void Assembler::esetzucc(Condition cc, Register dst) { 16557 assert(VM_Version::supports_apx_f(), ""); 16558 assert(0 <= cc && cc < 16, "illegal cc"); 16559 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 16560 // Encoding Format : eevex_prefix (4 bytes) | opcode_cc | modrm 16561 int encode = emit_eevex_prefix_ndd(dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3C /* MAP4 */, &attributes); // demotion disabled 16562 emit_opcode_prefix_and_encoding((0x40 | cc), 0xC0, encode); 16563 } 16564 16565 void Assembler::exorq(Register dst, Address src1, Register src2, bool no_flags) { 16566 InstructionMark im(this); 16567 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */); 16568 } 16569 16570 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) { 16571 if (VM_Version::supports_evex()) { 16572 _tuple_type = tuple_type; 16573 _input_size_in_bits = input_size_in_bits; 16574 } 16575 } 16576 16577 void Assembler::evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16578 assert(VM_Version::supports_avx512_vbmi() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16580 attributes.set_is_evex_instruction(); 16581 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16582 emit_int16(0x75, (0xC0 | encode)); 16583 } 16584 16585 void Assembler::evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16586 assert(VM_Version::supports_avx512bw() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16587 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16588 attributes.set_is_evex_instruction(); 16589 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16590 emit_int16(0x75, (0xC0 | encode)); 16591 } 16592 16593 void Assembler::evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16594 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16595 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16596 attributes.set_is_evex_instruction(); 16597 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16598 emit_int16(0x76, (0xC0 | encode)); 16599 } 16600 16601 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16602 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16603 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16604 attributes.set_is_evex_instruction(); 16605 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16606 emit_int16(0x76, (0xC0 | encode)); 16607 } 16608 16609 void Assembler::evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16610 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16611 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16612 attributes.set_is_evex_instruction(); 16613 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16614 emit_int16(0x77, (0xC0 | encode)); 16615 } 16616 16617 void Assembler::evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16618 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16619 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16620 attributes.set_is_evex_instruction(); 16621 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16622 emit_int16(0x77, (0xC0 | encode)); 16623 } 16624 16625 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16626 assert(VM_Version::supports_avx512_vbmi(), ""); 16627 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16628 attributes.set_is_evex_instruction(); 16629 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16630 emit_int16(0x7D, (0xC0 | encode)); 16631 } 16632 16633 void Assembler::evpermt2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16634 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx512vlbw() : VM_Version::supports_avx512bw(), ""); 16635 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16636 attributes.set_is_evex_instruction(); 16637 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16638 emit_int16(0x7D, (0xC0 | encode)); 16639 } 16640 16641 void Assembler::evpermt2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16642 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16643 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16644 attributes.set_is_evex_instruction(); 16645 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16646 emit_int16(0x7E, (0xC0 | encode)); 16647 } 16648 16649 void Assembler::evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16650 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); 16651 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16652 attributes.set_is_evex_instruction(); 16653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 16654 emit_int16(0x7E, (0xC0 | encode)); 16655 } 16656 16657 void Assembler::evaddph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16658 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16659 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16660 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16661 attributes.set_is_evex_instruction(); 16662 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16663 emit_int16(0x58, (0xC0 | encode)); 16664 } 16665 16666 void Assembler::evaddph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16667 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16668 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16669 InstructionMark im(this); 16670 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16671 attributes.set_is_evex_instruction(); 16672 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16673 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16674 emit_int8(0x58); 16675 emit_operand(dst, src, 0); 16676 } 16677 16678 void Assembler::evsubph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16679 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16680 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16682 attributes.set_is_evex_instruction(); 16683 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16684 emit_int16(0x5C, (0xC0 | encode)); 16685 } 16686 16687 void Assembler::evsubph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16688 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16689 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16690 InstructionMark im(this); 16691 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16692 attributes.set_is_evex_instruction(); 16693 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16694 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16695 emit_int8(0x5C); 16696 emit_operand(dst, src, 0); 16697 } 16698 16699 void Assembler::evmulph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16700 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16701 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16702 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16703 attributes.set_is_evex_instruction(); 16704 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16705 emit_int16(0x59, (0xC0 | encode)); 16706 } 16707 16708 void Assembler::evmulph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16709 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16710 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16711 InstructionMark im(this); 16712 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16713 attributes.set_is_evex_instruction(); 16714 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16715 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16716 emit_int8(0x59); 16717 emit_operand(dst, src, 0); 16718 } 16719 16720 void Assembler::evminph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16721 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16722 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16723 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16724 attributes.set_is_evex_instruction(); 16725 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16726 emit_int16(0x5D, (0xC0 | encode)); 16727 } 16728 16729 void Assembler::evminph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16730 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16731 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16732 InstructionMark im(this); 16733 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16734 attributes.set_is_evex_instruction(); 16735 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16736 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16737 emit_int8(0x5D); 16738 emit_operand(dst, src, 0); 16739 } 16740 16741 void Assembler::evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) { 16742 assert(VM_Version::supports_avx10_2(), ""); 16743 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 16744 attributes.set_is_evex_instruction(); 16745 attributes.set_embedded_opmask_register_specifier(mask); 16746 if (merge) { 16747 attributes.reset_is_clear_context(); 16748 } 16749 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes); 16750 emit_int24(0x52, (0xC0 | encode), imm8); 16751 } 16752 16753 void Assembler::evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) { 16754 assert(VM_Version::supports_avx10_2(), ""); 16755 InstructionMark im(this); 16756 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 16757 attributes.set_is_evex_instruction(); 16758 attributes.set_embedded_opmask_register_specifier(mask); 16759 if (merge) { 16760 attributes.reset_is_clear_context(); 16761 } 16762 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16763 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes); 16764 emit_int8(0x52); 16765 emit_operand(dst, src, 0); 16766 emit_int8(imm8); 16767 } 16768 16769 void Assembler::evmaxph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16770 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16771 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16772 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16773 attributes.set_is_evex_instruction(); 16774 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16775 emit_int16(0x5F, (0xC0 | encode)); 16776 } 16777 16778 void Assembler::evmaxph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16779 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16780 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16781 InstructionMark im(this); 16782 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16783 attributes.set_is_evex_instruction(); 16784 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16785 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16786 emit_int8(0x5F); 16787 emit_operand(dst, src, 0); 16788 } 16789 16790 void Assembler::evdivph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16791 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16792 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16793 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16794 attributes.set_is_evex_instruction(); 16795 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16796 emit_int16(0x5E, (0xC0 | encode)); 16797 } 16798 16799 void Assembler::evdivph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16800 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16"); 16801 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16802 InstructionMark im(this); 16803 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16804 attributes.set_is_evex_instruction(); 16805 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16806 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16807 emit_int8(0x5E); 16808 emit_operand(dst, src, 0); 16809 } 16810 16811 void Assembler::evsqrtph(XMMRegister dst, XMMRegister src, int vector_len) { 16812 assert(VM_Version::supports_avx512_fp16(), ""); 16813 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16814 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16815 attributes.set_is_evex_instruction(); 16816 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16817 emit_int16(0x51, (0xC0 | encode)); 16818 } 16819 16820 void Assembler::evsqrtph(XMMRegister dst, Address src, int vector_len) { 16821 assert(VM_Version::supports_avx512_fp16(), ""); 16822 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16823 InstructionMark im(this); 16824 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16825 attributes.set_is_evex_instruction(); 16826 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16827 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes); 16828 emit_int8(0x51); 16829 emit_operand(dst, src, 0); 16830 } 16831 16832 void Assembler::evfmadd132ph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 16833 assert(VM_Version::supports_avx512_fp16(), ""); 16834 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16835 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16836 attributes.set_is_evex_instruction(); 16837 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes); 16838 emit_int16(0x98, (0xC0 | encode)); 16839 } 16840 16841 void Assembler::evfmadd132ph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 16842 assert(VM_Version::supports_avx512_fp16(), ""); 16843 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), ""); 16844 InstructionMark im(this); 16845 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 16846 attributes.set_is_evex_instruction(); 16847 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 16848 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes); 16849 emit_int8(0x98); 16850 emit_operand(dst, src, 0); 16851 } 16852