1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "gc/shared/cardTableBarrierSet.hpp" 29 #include "gc/shared/collectedHeap.inline.hpp" 30 #include "interpreter/interpreter.hpp" 31 #include "memory/resourceArea.hpp" 32 #include "prims/methodHandles.hpp" 33 #include "runtime/objectMonitor.hpp" 34 #include "runtime/os.hpp" 35 #include "runtime/sharedRuntime.hpp" 36 #include "runtime/stubRoutines.hpp" 37 #include "runtime/vm_version.hpp" 38 #include "utilities/macros.hpp" 39 40 #ifdef PRODUCT 41 #define BLOCK_COMMENT(str) /* nothing */ 42 #define STOP(error) stop(error) 43 #else 44 #define BLOCK_COMMENT(str) block_comment(str) 45 #define STOP(error) block_comment(error); stop(error) 46 #endif 47 48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 49 // Implementation of AddressLiteral 50 51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms. 52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = { 53 // -----------------Table 4.5 -------------------- // 54 16, 32, 64, // EVEX_FV(0) 55 4, 4, 4, // EVEX_FV(1) - with Evex.b 56 16, 32, 64, // EVEX_FV(2) - with Evex.w 57 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b 58 8, 16, 32, // EVEX_HV(0) 59 4, 4, 4, // EVEX_HV(1) - with Evex.b 60 // -----------------Table 4.6 -------------------- // 61 16, 32, 64, // EVEX_FVM(0) 62 1, 1, 1, // EVEX_T1S(0) 63 2, 2, 2, // EVEX_T1S(1) 64 4, 4, 4, // EVEX_T1S(2) 65 8, 8, 8, // EVEX_T1S(3) 66 4, 4, 4, // EVEX_T1F(0) 67 8, 8, 8, // EVEX_T1F(1) 68 8, 8, 8, // EVEX_T2(0) 69 0, 16, 16, // EVEX_T2(1) 70 0, 16, 16, // EVEX_T4(0) 71 0, 0, 32, // EVEX_T4(1) 72 0, 0, 32, // EVEX_T8(0) 73 8, 16, 32, // EVEX_HVM(0) 74 4, 8, 16, // EVEX_QVM(0) 75 2, 4, 8, // EVEX_OVM(0) 76 16, 16, 16, // EVEX_M128(0) 77 8, 32, 64, // EVEX_DUP(0) 78 0, 0, 0 // EVEX_NTUP 79 }; 80 81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) { 82 _is_lval = false; 83 _target = target; 84 switch (rtype) { 85 case relocInfo::oop_type: 86 case relocInfo::metadata_type: 87 // Oops are a special case. Normally they would be their own section 88 // but in cases like icBuffer they are literals in the code stream that 89 // we don't have a section for. We use none so that we get a literal address 90 // which is always patchable. 91 break; 92 case relocInfo::external_word_type: 93 _rspec = external_word_Relocation::spec(target); 94 break; 95 case relocInfo::internal_word_type: 96 _rspec = internal_word_Relocation::spec(target); 97 break; 98 case relocInfo::opt_virtual_call_type: 99 _rspec = opt_virtual_call_Relocation::spec(); 100 break; 101 case relocInfo::static_call_type: 102 _rspec = static_call_Relocation::spec(); 103 break; 104 case relocInfo::runtime_call_type: 105 _rspec = runtime_call_Relocation::spec(); 106 break; 107 case relocInfo::poll_type: 108 case relocInfo::poll_return_type: 109 _rspec = Relocation::spec_simple(rtype); 110 break; 111 case relocInfo::none: 112 break; 113 default: 114 ShouldNotReachHere(); 115 break; 116 } 117 } 118 119 // Implementation of Address 120 121 #ifdef _LP64 122 123 Address Address::make_array(ArrayAddress adr) { 124 // Not implementable on 64bit machines 125 // Should have been handled higher up the call chain. 126 ShouldNotReachHere(); 127 return Address(); 128 } 129 130 // exceedingly dangerous constructor 131 Address::Address(int disp, address loc, relocInfo::relocType rtype) { 132 _base = noreg; 133 _index = noreg; 134 _scale = no_scale; 135 _disp = disp; 136 _xmmindex = xnoreg; 137 _isxmmindex = false; 138 switch (rtype) { 139 case relocInfo::external_word_type: 140 _rspec = external_word_Relocation::spec(loc); 141 break; 142 case relocInfo::internal_word_type: 143 _rspec = internal_word_Relocation::spec(loc); 144 break; 145 case relocInfo::runtime_call_type: 146 // HMM 147 _rspec = runtime_call_Relocation::spec(); 148 break; 149 case relocInfo::poll_type: 150 case relocInfo::poll_return_type: 151 _rspec = Relocation::spec_simple(rtype); 152 break; 153 case relocInfo::none: 154 break; 155 default: 156 ShouldNotReachHere(); 157 } 158 } 159 #else // LP64 160 161 Address Address::make_array(ArrayAddress adr) { 162 AddressLiteral base = adr.base(); 163 Address index = adr.index(); 164 assert(index._disp == 0, "must not have disp"); // maybe it can? 165 Address array(index._base, index._index, index._scale, (intptr_t) base.target()); 166 array._rspec = base._rspec; 167 return array; 168 } 169 170 // exceedingly dangerous constructor 171 Address::Address(address loc, RelocationHolder spec) { 172 _base = noreg; 173 _index = noreg; 174 _scale = no_scale; 175 _disp = (intptr_t) loc; 176 _rspec = spec; 177 _xmmindex = xnoreg; 178 _isxmmindex = false; 179 } 180 181 #endif // _LP64 182 183 184 185 // Convert the raw encoding form into the form expected by the constructor for 186 // Address. An index of 4 (rsp) corresponds to having no index, so convert 187 // that to noreg for the Address constructor. 188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) { 189 RelocationHolder rspec = RelocationHolder::none; 190 if (disp_reloc != relocInfo::none) { 191 rspec = Relocation::spec_simple(disp_reloc); 192 } 193 bool valid_index = index != rsp->encoding(); 194 if (valid_index) { 195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp)); 196 madr._rspec = rspec; 197 return madr; 198 } else { 199 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp)); 200 madr._rspec = rspec; 201 return madr; 202 } 203 } 204 205 // Implementation of Assembler 206 207 int AbstractAssembler::code_fill_byte() { 208 return (u_char)'\xF4'; // hlt 209 } 210 211 void Assembler::init_attributes(void) { 212 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false); 213 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false); 214 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false); 215 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false); 216 NOT_LP64(_is_managed = false;) 217 _attributes = NULL; 218 } 219 220 221 void Assembler::membar(Membar_mask_bits order_constraint) { 222 // We only have to handle StoreLoad 223 if (order_constraint & StoreLoad) { 224 // All usable chips support "locked" instructions which suffice 225 // as barriers, and are much faster than the alternative of 226 // using cpuid instruction. We use here a locked add [esp-C],0. 227 // This is conveniently otherwise a no-op except for blowing 228 // flags, and introducing a false dependency on target memory 229 // location. We can't do anything with flags, but we can avoid 230 // memory dependencies in the current method by locked-adding 231 // somewhere else on the stack. Doing [esp+C] will collide with 232 // something on stack in current method, hence we go for [esp-C]. 233 // It is convenient since it is almost always in data cache, for 234 // any small C. We need to step back from SP to avoid data 235 // dependencies with other things on below SP (callee-saves, for 236 // example). Without a clear way to figure out the minimal safe 237 // distance from SP, it makes sense to step back the complete 238 // cache line, as this will also avoid possible second-order effects 239 // with locked ops against the cache line. Our choice of offset 240 // is bounded by x86 operand encoding, which should stay within 241 // [-128; +127] to have the 8-byte displacement encoding. 242 // 243 // Any change to this code may need to revisit other places in 244 // the code where this idiom is used, in particular the 245 // orderAccess code. 246 247 int offset = -VM_Version::L1_line_size(); 248 if (offset < -128) { 249 offset = -128; 250 } 251 252 lock(); 253 addl(Address(rsp, offset), 0);// Assert the lock# signal here 254 } 255 } 256 257 // make this go away someday 258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) { 259 if (rtype == relocInfo::none) 260 emit_int32(data); 261 else 262 emit_data(data, Relocation::spec_simple(rtype), format); 263 } 264 265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) { 266 assert(imm_operand == 0, "default format must be immediate in this file"); 267 assert(inst_mark() != NULL, "must be inside InstructionMark"); 268 if (rspec.type() != relocInfo::none) { 269 #ifdef ASSERT 270 check_relocation(rspec, format); 271 #endif 272 // Do not use AbstractAssembler::relocate, which is not intended for 273 // embedded words. Instead, relocate to the enclosing instruction. 274 275 // hack. call32 is too wide for mask so use disp32 276 if (format == call32_operand) 277 code_section()->relocate(inst_mark(), rspec, disp32_operand); 278 else 279 code_section()->relocate(inst_mark(), rspec, format); 280 } 281 emit_int32(data); 282 } 283 284 static int encode(Register r) { 285 int enc = r->encoding(); 286 if (enc >= 8) { 287 enc -= 8; 288 } 289 return enc; 290 } 291 292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) { 293 assert(dst->has_byte_register(), "must have byte register"); 294 assert(isByte(op1) && isByte(op2), "wrong opcode"); 295 assert(isByte(imm8), "not a byte"); 296 assert((op1 & 0x01) == 0, "should be 8bit operation"); 297 emit_int24(op1, (op2 | encode(dst)), imm8); 298 } 299 300 301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) { 302 assert(isByte(op1) && isByte(op2), "wrong opcode"); 303 assert(op1 == 0x81, "Unexpected opcode"); 304 if (is8bit(imm32)) { 305 emit_int24(op1 | 0x02, // set sign bit 306 op2 | encode(dst), 307 imm32 & 0xFF); 308 } else if (dst == rax) { 309 switch (op2) { 310 case 0xD0: emit_int8(0x15); break; // adc 311 case 0xC0: emit_int8(0x05); break; // add 312 case 0xE0: emit_int8(0x25); break; // and 313 case 0xF8: emit_int8(0x3D); break; // cmp 314 case 0xC8: emit_int8(0x0D); break; // or 315 case 0xD8: emit_int8(0x1D); break; // sbb 316 case 0xE8: emit_int8(0x2D); break; // sub 317 case 0xF0: emit_int8(0x35); break; // xor 318 default: ShouldNotReachHere(); 319 } 320 emit_int32(imm32); 321 } else { 322 emit_int16(op1, (op2 | encode(dst))); 323 emit_int32(imm32); 324 } 325 } 326 327 // Force generation of a 4 byte immediate value even if it fits into 8bit 328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) { 329 assert(isByte(op1) && isByte(op2), "wrong opcode"); 330 assert((op1 & 0x01) == 1, "should be 32bit operation"); 331 assert((op1 & 0x02) == 0, "sign-extension bit should not be set"); 332 emit_int16(op1, (op2 | encode(dst))); 333 emit_int32(imm32); 334 } 335 336 // immediate-to-memory forms 337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) { 338 assert((op1 & 0x01) == 1, "should be 32bit operation"); 339 assert((op1 & 0x02) == 0, "sign-extension bit should not be set"); 340 if (is8bit(imm32)) { 341 emit_int8(op1 | 0x02); // set sign bit 342 emit_operand(rm, adr, 1); 343 emit_int8(imm32 & 0xFF); 344 } else { 345 emit_int8(op1); 346 emit_operand(rm, adr, 4); 347 emit_int32(imm32); 348 } 349 } 350 351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) { 352 assert(op1 == 0x81, "unexpected opcode"); 353 emit_int8(op1); 354 emit_operand(rm, adr, 4); 355 emit_int32(imm32); 356 } 357 358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) { 359 assert(isByte(op1) && isByte(op2), "wrong opcode"); 360 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src))); 361 } 362 363 364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len, 365 int cur_tuple_type, int in_size_in_bits, int cur_encoding) { 366 int mod_idx = 0; 367 // We will test if the displacement fits the compressed format and if so 368 // apply the compression to the displacement iff the result is8bit. 369 if (VM_Version::supports_evex() && is_evex_inst) { 370 switch (cur_tuple_type) { 371 case EVEX_FV: 372 if ((cur_encoding & VEX_W) == VEX_W) { 373 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2; 374 } else { 375 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 376 } 377 break; 378 379 case EVEX_HV: 380 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 381 break; 382 383 case EVEX_FVM: 384 break; 385 386 case EVEX_T1S: 387 switch (in_size_in_bits) { 388 case EVEX_8bit: 389 break; 390 391 case EVEX_16bit: 392 mod_idx = 1; 393 break; 394 395 case EVEX_32bit: 396 mod_idx = 2; 397 break; 398 399 case EVEX_64bit: 400 mod_idx = 3; 401 break; 402 } 403 break; 404 405 case EVEX_T1F: 406 case EVEX_T2: 407 case EVEX_T4: 408 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0; 409 break; 410 411 case EVEX_T8: 412 break; 413 414 case EVEX_HVM: 415 break; 416 417 case EVEX_QVM: 418 break; 419 420 case EVEX_OVM: 421 break; 422 423 case EVEX_M128: 424 break; 425 426 case EVEX_DUP: 427 break; 428 429 default: 430 assert(0, "no valid evex tuple_table entry"); 431 break; 432 } 433 434 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) { 435 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len]; 436 if ((disp % disp_factor) == 0) { 437 int new_disp = disp / disp_factor; 438 if ((-0x80 <= new_disp && new_disp < 0x80)) { 439 disp = new_disp; 440 } 441 } else { 442 return false; 443 } 444 } 445 } 446 return (-0x80 <= disp && disp < 0x80); 447 } 448 449 450 bool Assembler::emit_compressed_disp_byte(int &disp) { 451 int mod_idx = 0; 452 // We will test if the displacement fits the compressed format and if so 453 // apply the compression to the displacement iff the result is8bit. 454 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) { 455 int evex_encoding = _attributes->get_evex_encoding(); 456 int tuple_type = _attributes->get_tuple_type(); 457 switch (tuple_type) { 458 case EVEX_FV: 459 if ((evex_encoding & VEX_W) == VEX_W) { 460 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2; 461 } else { 462 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 463 } 464 break; 465 466 case EVEX_HV: 467 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0; 468 break; 469 470 case EVEX_FVM: 471 break; 472 473 case EVEX_T1S: 474 switch (_attributes->get_input_size()) { 475 case EVEX_8bit: 476 break; 477 478 case EVEX_16bit: 479 mod_idx = 1; 480 break; 481 482 case EVEX_32bit: 483 mod_idx = 2; 484 break; 485 486 case EVEX_64bit: 487 mod_idx = 3; 488 break; 489 } 490 break; 491 492 case EVEX_T1F: 493 case EVEX_T2: 494 case EVEX_T4: 495 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0; 496 break; 497 498 case EVEX_T8: 499 break; 500 501 case EVEX_HVM: 502 break; 503 504 case EVEX_QVM: 505 break; 506 507 case EVEX_OVM: 508 break; 509 510 case EVEX_M128: 511 break; 512 513 case EVEX_DUP: 514 break; 515 516 default: 517 assert(0, "no valid evex tuple_table entry"); 518 break; 519 } 520 521 int vector_len = _attributes->get_vector_len(); 522 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) { 523 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len]; 524 if ((disp % disp_factor) == 0) { 525 int new_disp = disp / disp_factor; 526 if (is8bit(new_disp)) { 527 disp = new_disp; 528 } 529 } else { 530 return false; 531 } 532 } 533 } 534 return is8bit(disp); 535 } 536 537 static bool is_valid_encoding(int reg_enc) { 538 return reg_enc >= 0; 539 } 540 541 static int raw_encode(Register reg) { 542 assert(reg == noreg || reg->is_valid(), "sanity"); 543 int reg_enc = reg->raw_encoding(); 544 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity"); 545 return reg_enc; 546 } 547 548 static int raw_encode(XMMRegister xmmreg) { 549 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity"); 550 int xmmreg_enc = xmmreg->raw_encoding(); 551 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity"); 552 return xmmreg_enc; 553 } 554 555 static int raw_encode(KRegister kreg) { 556 assert(kreg == knoreg || kreg->is_valid(), "sanity"); 557 int kreg_enc = kreg->raw_encoding(); 558 assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity"); 559 return kreg_enc; 560 } 561 562 static int modrm_encoding(int mod, int dst_enc, int src_enc) { 563 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7); 564 } 565 566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) { 567 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7); 568 } 569 570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) { 571 assert((mod & 3) != 0b11, "forbidden"); 572 int modrm = modrm_encoding(mod, dst_enc, src_enc); 573 emit_int8(modrm); 574 } 575 576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc, 577 int disp) { 578 int modrm = modrm_encoding(mod, dst_enc, src_enc); 579 emit_int16(modrm, disp & 0xFF); 580 } 581 582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc, 583 Address::ScaleFactor scale, int index_enc, int base_enc) { 584 int modrm = modrm_encoding(mod, dst_enc, src_enc); 585 int sib = sib_encoding(scale, index_enc, base_enc); 586 emit_int16(modrm, sib); 587 } 588 589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc, 590 Address::ScaleFactor scale, int index_enc, int base_enc, 591 int disp) { 592 int modrm = modrm_encoding(mod, dst_enc, src_enc); 593 int sib = sib_encoding(scale, index_enc, base_enc); 594 emit_int24(modrm, sib, disp & 0xFF); 595 } 596 597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc, 598 Address::ScaleFactor scale, int disp, 599 RelocationHolder const& rspec, 600 int post_addr_length) { 601 bool no_relocation = (rspec.type() == relocInfo::none); 602 603 if (is_valid_encoding(base_enc)) { 604 if (is_valid_encoding(index_enc)) { 605 assert(scale != Address::no_scale, "inconsistent address"); 606 // [base + index*scale + disp] 607 if (disp == 0 && no_relocation && 608 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) { 609 // [base + index*scale] 610 // [00 reg 100][ss index base] 611 emit_modrm_sib(0b00, reg_enc, 0b100, 612 scale, index_enc, base_enc); 613 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 614 // [base + index*scale + imm8] 615 // [01 reg 100][ss index base] imm8 616 emit_modrm_sib_disp8(0b01, reg_enc, 0b100, 617 scale, index_enc, base_enc, 618 disp); 619 } else { 620 // [base + index*scale + disp32] 621 // [10 reg 100][ss index base] disp32 622 emit_modrm_sib(0b10, reg_enc, 0b100, 623 scale, index_enc, base_enc); 624 emit_data(disp, rspec, disp32_operand); 625 } 626 } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) { 627 // [rsp + disp] 628 if (disp == 0 && no_relocation) { 629 // [rsp] 630 // [00 reg 100][00 100 100] 631 emit_modrm_sib(0b00, reg_enc, 0b100, 632 Address::times_1, 0b100, 0b100); 633 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 634 // [rsp + imm8] 635 // [01 reg 100][00 100 100] disp8 636 emit_modrm_sib_disp8(0b01, reg_enc, 0b100, 637 Address::times_1, 0b100, 0b100, 638 disp); 639 } else { 640 // [rsp + imm32] 641 // [10 reg 100][00 100 100] disp32 642 emit_modrm_sib(0b10, reg_enc, 0b100, 643 Address::times_1, 0b100, 0b100); 644 emit_data(disp, rspec, disp32_operand); 645 } 646 } else { 647 // [base + disp] 648 assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode"); 649 if (disp == 0 && no_relocation && 650 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) { 651 // [base] 652 // [00 reg base] 653 emit_modrm(0, reg_enc, base_enc); 654 } else if (emit_compressed_disp_byte(disp) && no_relocation) { 655 // [base + disp8] 656 // [01 reg base] disp8 657 emit_modrm_disp8(0b01, reg_enc, base_enc, 658 disp); 659 } else { 660 // [base + disp32] 661 // [10 reg base] disp32 662 emit_modrm(0b10, reg_enc, base_enc); 663 emit_data(disp, rspec, disp32_operand); 664 } 665 } 666 } else { 667 if (is_valid_encoding(index_enc)) { 668 assert(scale != Address::no_scale, "inconsistent address"); 669 // base == noreg 670 // [index*scale + disp] 671 // [00 reg 100][ss index 101] disp32 672 emit_modrm_sib(0b00, reg_enc, 0b100, 673 scale, index_enc, 0b101 /* no base */); 674 emit_data(disp, rspec, disp32_operand); 675 } else if (!no_relocation) { 676 // base == noreg, index == noreg 677 // [disp] (64bit) RIP-RELATIVE (32bit) abs 678 // [00 reg 101] disp32 679 680 emit_modrm(0b00, reg_enc, 0b101 /* no base */); 681 // Note that the RIP-rel. correction applies to the generated 682 // disp field, but _not_ to the target address in the rspec. 683 684 // disp was created by converting the target address minus the pc 685 // at the start of the instruction. That needs more correction here. 686 // intptr_t disp = target - next_ip; 687 assert(inst_mark() != NULL, "must be inside InstructionMark"); 688 address next_ip = pc() + sizeof(int32_t) + post_addr_length; 689 int64_t adjusted = disp; 690 // Do rip-rel adjustment for 64bit 691 LP64_ONLY(adjusted -= (next_ip - inst_mark())); 692 assert(is_simm32(adjusted), 693 "must be 32bit offset (RIP relative address)"); 694 emit_data((int32_t) adjusted, rspec, disp32_operand); 695 696 } else { 697 // base == noreg, index == noreg, no_relocation == true 698 // 32bit never did this, did everything as the rip-rel/disp code above 699 // [disp] ABSOLUTE 700 // [00 reg 100][00 100 101] disp32 701 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */, 702 Address::times_1, 0b100, 0b101); 703 emit_data(disp, rspec, disp32_operand); 704 } 705 } 706 } 707 708 void Assembler::emit_operand(Register reg, Register base, Register index, 709 Address::ScaleFactor scale, int disp, 710 RelocationHolder const& rspec, 711 int post_addr_length) { 712 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 713 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index), 714 scale, disp, rspec, post_addr_length); 715 716 } 717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index, 718 Address::ScaleFactor scale, int disp, 719 RelocationHolder const& rspec, 720 int post_addr_length) { 721 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 722 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported"); 723 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index), 724 scale, disp, rspec, post_addr_length); 725 } 726 727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex, 728 Address::ScaleFactor scale, int disp, 729 RelocationHolder const& rspec, 730 int post_addr_length) { 731 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported"); 732 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported"); 733 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex), 734 scale, disp, rspec, post_addr_length); 735 } 736 737 void Assembler::emit_operand(KRegister kreg, Address adr, 738 int post_addr_length) { 739 emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp, 740 adr._rspec, 741 post_addr_length); 742 } 743 744 void Assembler::emit_operand(KRegister kreg, Register base, Register index, 745 Address::ScaleFactor scale, int disp, 746 RelocationHolder const& rspec, 747 int post_addr_length) { 748 assert(!index->is_valid() || index != rsp, "illegal addressing mode"); 749 emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index), 750 scale, disp, rspec, post_addr_length); 751 } 752 753 // Secret local extension to Assembler::WhichOperand: 754 #define end_pc_operand (_WhichOperand_limit) 755 756 address Assembler::locate_operand(address inst, WhichOperand which) { 757 // Decode the given instruction, and return the address of 758 // an embedded 32-bit operand word. 759 760 // If "which" is disp32_operand, selects the displacement portion 761 // of an effective address specifier. 762 // If "which" is imm64_operand, selects the trailing immediate constant. 763 // If "which" is call32_operand, selects the displacement of a call or jump. 764 // Caller is responsible for ensuring that there is such an operand, 765 // and that it is 32/64 bits wide. 766 767 // If "which" is end_pc_operand, find the end of the instruction. 768 769 address ip = inst; 770 bool is_64bit = false; 771 772 debug_only(bool has_disp32 = false); 773 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn 774 775 again_after_prefix: 776 switch (0xFF & *ip++) { 777 778 // These convenience macros generate groups of "case" labels for the switch. 779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3 780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \ 781 case (x)+4: case (x)+5: case (x)+6: case (x)+7 782 #define REP16(x) REP8((x)+0): \ 783 case REP8((x)+8) 784 785 case CS_segment: 786 case SS_segment: 787 case DS_segment: 788 case ES_segment: 789 case FS_segment: 790 case GS_segment: 791 // Seems dubious 792 LP64_ONLY(assert(false, "shouldn't have that prefix")); 793 assert(ip == inst+1, "only one prefix allowed"); 794 goto again_after_prefix; 795 796 case 0x67: 797 case REX: 798 case REX_B: 799 case REX_X: 800 case REX_XB: 801 case REX_R: 802 case REX_RB: 803 case REX_RX: 804 case REX_RXB: 805 NOT_LP64(assert(false, "64bit prefixes")); 806 goto again_after_prefix; 807 808 case REX_W: 809 case REX_WB: 810 case REX_WX: 811 case REX_WXB: 812 case REX_WR: 813 case REX_WRB: 814 case REX_WRX: 815 case REX_WRXB: 816 NOT_LP64(assert(false, "64bit prefixes")); 817 is_64bit = true; 818 goto again_after_prefix; 819 820 case 0xFF: // pushq a; decl a; incl a; call a; jmp a 821 case 0x88: // movb a, r 822 case 0x89: // movl a, r 823 case 0x8A: // movb r, a 824 case 0x8B: // movl r, a 825 case 0x8F: // popl a 826 debug_only(has_disp32 = true); 827 break; 828 829 case 0x68: // pushq #32 830 if (which == end_pc_operand) { 831 return ip + 4; 832 } 833 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate"); 834 return ip; // not produced by emit_operand 835 836 case 0x66: // movw ... (size prefix) 837 again_after_size_prefix2: 838 switch (0xFF & *ip++) { 839 case REX: 840 case REX_B: 841 case REX_X: 842 case REX_XB: 843 case REX_R: 844 case REX_RB: 845 case REX_RX: 846 case REX_RXB: 847 case REX_W: 848 case REX_WB: 849 case REX_WX: 850 case REX_WXB: 851 case REX_WR: 852 case REX_WRB: 853 case REX_WRX: 854 case REX_WRXB: 855 NOT_LP64(assert(false, "64bit prefix found")); 856 goto again_after_size_prefix2; 857 case 0x8B: // movw r, a 858 case 0x89: // movw a, r 859 debug_only(has_disp32 = true); 860 break; 861 case 0xC7: // movw a, #16 862 debug_only(has_disp32 = true); 863 tail_size = 2; // the imm16 864 break; 865 case 0x0F: // several SSE/SSE2 variants 866 ip--; // reparse the 0x0F 867 goto again_after_prefix; 868 default: 869 ShouldNotReachHere(); 870 } 871 break; 872 873 case REP8(0xB8): // movl/q r, #32/#64(oop?) 874 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4); 875 // these asserts are somewhat nonsensical 876 #ifndef _LP64 877 assert(which == imm_operand || which == disp32_operand, 878 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)); 879 #else 880 assert((which == call32_operand || which == imm_operand) && is_64bit || 881 which == narrow_oop_operand && !is_64bit, 882 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)); 883 #endif // _LP64 884 return ip; 885 886 case 0x69: // imul r, a, #32 887 case 0xC7: // movl a, #32(oop?) 888 tail_size = 4; 889 debug_only(has_disp32 = true); // has both kinds of operands! 890 break; 891 892 case 0x0F: // movx..., etc. 893 switch (0xFF & *ip++) { 894 case 0x3A: // pcmpestri 895 tail_size = 1; 896 case 0x38: // ptest, pmovzxbw 897 ip++; // skip opcode 898 debug_only(has_disp32 = true); // has both kinds of operands! 899 break; 900 901 case 0x70: // pshufd r, r/a, #8 902 debug_only(has_disp32 = true); // has both kinds of operands! 903 case 0x73: // psrldq r, #8 904 tail_size = 1; 905 break; 906 907 case 0x10: // movups 908 case 0x11: // movups 909 case 0x12: // movlps 910 case 0x28: // movaps 911 case 0x2E: // ucomiss 912 case 0x2F: // comiss 913 case 0x54: // andps 914 case 0x55: // andnps 915 case 0x56: // orps 916 case 0x57: // xorps 917 case 0x58: // addpd 918 case 0x59: // mulpd 919 case 0x6E: // movd 920 case 0x7E: // movd 921 case 0x6F: // movdq 922 case 0x7F: // movdq 923 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush 924 case 0xFE: // paddd 925 debug_only(has_disp32 = true); 926 break; 927 928 case 0xAD: // shrd r, a, %cl 929 case 0xAF: // imul r, a 930 case 0xBE: // movsbl r, a (movsxb) 931 case 0xBF: // movswl r, a (movsxw) 932 case 0xB6: // movzbl r, a (movzxb) 933 case 0xB7: // movzwl r, a (movzxw) 934 case REP16(0x40): // cmovl cc, r, a 935 case 0xB0: // cmpxchgb 936 case 0xB1: // cmpxchg 937 case 0xC1: // xaddl 938 case 0xC7: // cmpxchg8 939 case REP16(0x90): // setcc a 940 debug_only(has_disp32 = true); 941 // fall out of the switch to decode the address 942 break; 943 944 case 0xC4: // pinsrw r, a, #8 945 debug_only(has_disp32 = true); 946 case 0xC5: // pextrw r, r, #8 947 tail_size = 1; // the imm8 948 break; 949 950 case 0xAC: // shrd r, a, #8 951 debug_only(has_disp32 = true); 952 tail_size = 1; // the imm8 953 break; 954 955 case REP16(0x80): // jcc rdisp32 956 if (which == end_pc_operand) return ip + 4; 957 assert(which == call32_operand, "jcc has no disp32 or imm"); 958 return ip; 959 default: 960 ShouldNotReachHere(); 961 } 962 break; 963 964 case 0x81: // addl a, #32; addl r, #32 965 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl 966 // on 32bit in the case of cmpl, the imm might be an oop 967 tail_size = 4; 968 debug_only(has_disp32 = true); // has both kinds of operands! 969 break; 970 971 case 0x83: // addl a, #8; addl r, #8 972 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl 973 debug_only(has_disp32 = true); // has both kinds of operands! 974 tail_size = 1; 975 break; 976 977 case 0x15: // adc rax, #32 978 case 0x05: // add rax, #32 979 case 0x25: // and rax, #32 980 case 0x3D: // cmp rax, #32 981 case 0x0D: // or rax, #32 982 case 0x1D: // sbb rax, #32 983 case 0x2D: // sub rax, #32 984 case 0x35: // xor rax, #32 985 return which == end_pc_operand ? ip + 4 : ip; 986 987 case 0x9B: 988 switch (0xFF & *ip++) { 989 case 0xD9: // fnstcw a 990 debug_only(has_disp32 = true); 991 break; 992 default: 993 ShouldNotReachHere(); 994 } 995 break; 996 997 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a 998 case REP4(0x10): // adc... 999 case REP4(0x20): // and... 1000 case REP4(0x30): // xor... 1001 case REP4(0x08): // or... 1002 case REP4(0x18): // sbb... 1003 case REP4(0x28): // sub... 1004 case 0xF7: // mull a 1005 case 0x8D: // lea r, a 1006 case 0x87: // xchg r, a 1007 case REP4(0x38): // cmp... 1008 case 0x85: // test r, a 1009 debug_only(has_disp32 = true); // has both kinds of operands! 1010 break; 1011 1012 case 0xA8: // testb rax, #8 1013 return which == end_pc_operand ? ip + 1 : ip; 1014 case 0xA9: // testl/testq rax, #32 1015 return which == end_pc_operand ? ip + 4 : ip; 1016 1017 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8 1018 case 0xC6: // movb a, #8 1019 case 0x80: // cmpb a, #8 1020 case 0x6B: // imul r, a, #8 1021 debug_only(has_disp32 = true); // has both kinds of operands! 1022 tail_size = 1; // the imm8 1023 break; 1024 1025 case 0xC4: // VEX_3bytes 1026 case 0xC5: // VEX_2bytes 1027 assert((UseAVX > 0), "shouldn't have VEX prefix"); 1028 assert(ip == inst+1, "no prefixes allowed"); 1029 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions 1030 // but they have prefix 0x0F and processed when 0x0F processed above. 1031 // 1032 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES 1033 // instructions (these instructions are not supported in 64-bit mode). 1034 // To distinguish them bits [7:6] are set in the VEX second byte since 1035 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set 1036 // those VEX bits REX and vvvv bits are inverted. 1037 // 1038 // Fortunately C2 doesn't generate these instructions so we don't need 1039 // to check for them in product version. 1040 1041 // Check second byte 1042 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions")); 1043 1044 int vex_opcode; 1045 // First byte 1046 if ((0xFF & *inst) == VEX_3bytes) { 1047 vex_opcode = VEX_OPCODE_MASK & *ip; 1048 ip++; // third byte 1049 is_64bit = ((VEX_W & *ip) == VEX_W); 1050 } else { 1051 vex_opcode = VEX_OPCODE_0F; 1052 } 1053 ip++; // opcode 1054 // To find the end of instruction (which == end_pc_operand). 1055 switch (vex_opcode) { 1056 case VEX_OPCODE_0F: 1057 switch (0xFF & *ip) { 1058 case 0x70: // pshufd r, r/a, #8 1059 case 0x71: // ps[rl|ra|ll]w r, #8 1060 case 0x72: // ps[rl|ra|ll]d r, #8 1061 case 0x73: // ps[rl|ra|ll]q r, #8 1062 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8 1063 case 0xC4: // pinsrw r, r, r/a, #8 1064 case 0xC5: // pextrw r/a, r, #8 1065 case 0xC6: // shufp[s|d] r, r, r/a, #8 1066 tail_size = 1; // the imm8 1067 break; 1068 } 1069 break; 1070 case VEX_OPCODE_0F_3A: 1071 tail_size = 1; 1072 break; 1073 } 1074 ip++; // skip opcode 1075 debug_only(has_disp32 = true); // has both kinds of operands! 1076 break; 1077 1078 case 0x62: // EVEX_4bytes 1079 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix"); 1080 assert(ip == inst+1, "no prefixes allowed"); 1081 // no EVEX collisions, all instructions that have 0x62 opcodes 1082 // have EVEX versions and are subopcodes of 0x66 1083 ip++; // skip P0 and examine W in P1 1084 is_64bit = ((VEX_W & *ip) == VEX_W); 1085 ip++; // move to P2 1086 ip++; // skip P2, move to opcode 1087 // To find the end of instruction (which == end_pc_operand). 1088 switch (0xFF & *ip) { 1089 case 0x22: // pinsrd r, r/a, #8 1090 case 0x61: // pcmpestri r, r/a, #8 1091 case 0x70: // pshufd r, r/a, #8 1092 case 0x73: // psrldq r, #8 1093 case 0x1f: // evpcmpd/evpcmpq 1094 case 0x3f: // evpcmpb/evpcmpw 1095 tail_size = 1; // the imm8 1096 break; 1097 default: 1098 break; 1099 } 1100 ip++; // skip opcode 1101 debug_only(has_disp32 = true); // has both kinds of operands! 1102 break; 1103 1104 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1 1105 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl 1106 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a 1107 case 0xDD: // fld_d a; fst_d a; fstp_d a 1108 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a 1109 case 0xDF: // fild_d a; fistp_d a 1110 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a 1111 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a 1112 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a 1113 debug_only(has_disp32 = true); 1114 break; 1115 1116 case 0xE8: // call rdisp32 1117 case 0xE9: // jmp rdisp32 1118 if (which == end_pc_operand) return ip + 4; 1119 assert(which == call32_operand, "call has no disp32 or imm"); 1120 return ip; 1121 1122 case 0xF0: // Lock 1123 goto again_after_prefix; 1124 1125 case 0xF3: // For SSE 1126 case 0xF2: // For SSE2 1127 switch (0xFF & *ip++) { 1128 case REX: 1129 case REX_B: 1130 case REX_X: 1131 case REX_XB: 1132 case REX_R: 1133 case REX_RB: 1134 case REX_RX: 1135 case REX_RXB: 1136 case REX_W: 1137 case REX_WB: 1138 case REX_WX: 1139 case REX_WXB: 1140 case REX_WR: 1141 case REX_WRB: 1142 case REX_WRX: 1143 case REX_WRXB: 1144 NOT_LP64(assert(false, "found 64bit prefix")); 1145 ip++; 1146 default: 1147 ip++; 1148 } 1149 debug_only(has_disp32 = true); // has both kinds of operands! 1150 break; 1151 1152 default: 1153 ShouldNotReachHere(); 1154 1155 #undef REP8 1156 #undef REP16 1157 } 1158 1159 assert(which != call32_operand, "instruction is not a call, jmp, or jcc"); 1160 #ifdef _LP64 1161 assert(which != imm_operand, "instruction is not a movq reg, imm64"); 1162 #else 1163 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field"); 1164 assert(which != imm_operand || has_disp32, "instruction has no imm32 field"); 1165 #endif // LP64 1166 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field"); 1167 1168 // parse the output of emit_operand 1169 int op2 = 0xFF & *ip++; 1170 int base = op2 & 0x07; 1171 int op3 = -1; 1172 const int b100 = 4; 1173 const int b101 = 5; 1174 if (base == b100 && (op2 >> 6) != 3) { 1175 op3 = 0xFF & *ip++; 1176 base = op3 & 0x07; // refetch the base 1177 } 1178 // now ip points at the disp (if any) 1179 1180 switch (op2 >> 6) { 1181 case 0: 1182 // [00 reg 100][ss index base] 1183 // [00 reg 100][00 100 esp] 1184 // [00 reg base] 1185 // [00 reg 100][ss index 101][disp32] 1186 // [00 reg 101] [disp32] 1187 1188 if (base == b101) { 1189 if (which == disp32_operand) 1190 return ip; // caller wants the disp32 1191 ip += 4; // skip the disp32 1192 } 1193 break; 1194 1195 case 1: 1196 // [01 reg 100][ss index base][disp8] 1197 // [01 reg 100][00 100 esp][disp8] 1198 // [01 reg base] [disp8] 1199 ip += 1; // skip the disp8 1200 break; 1201 1202 case 2: 1203 // [10 reg 100][ss index base][disp32] 1204 // [10 reg 100][00 100 esp][disp32] 1205 // [10 reg base] [disp32] 1206 if (which == disp32_operand) 1207 return ip; // caller wants the disp32 1208 ip += 4; // skip the disp32 1209 break; 1210 1211 case 3: 1212 // [11 reg base] (not a memory addressing mode) 1213 break; 1214 } 1215 1216 if (which == end_pc_operand) { 1217 return ip + tail_size; 1218 } 1219 1220 #ifdef _LP64 1221 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32"); 1222 #else 1223 assert(which == imm_operand, "instruction has only an imm field"); 1224 #endif // LP64 1225 return ip; 1226 } 1227 1228 address Assembler::locate_next_instruction(address inst) { 1229 // Secretly share code with locate_operand: 1230 return locate_operand(inst, end_pc_operand); 1231 } 1232 1233 1234 #ifdef ASSERT 1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) { 1236 address inst = inst_mark(); 1237 assert(inst != NULL && inst < pc(), "must point to beginning of instruction"); 1238 address opnd; 1239 1240 Relocation* r = rspec.reloc(); 1241 if (r->type() == relocInfo::none) { 1242 return; 1243 } else if (r->is_call() || format == call32_operand) { 1244 // assert(format == imm32_operand, "cannot specify a nonzero format"); 1245 opnd = locate_operand(inst, call32_operand); 1246 } else if (r->is_data()) { 1247 assert(format == imm_operand || format == disp32_operand 1248 LP64_ONLY(|| format == narrow_oop_operand), "format ok"); 1249 opnd = locate_operand(inst, (WhichOperand)format); 1250 } else { 1251 assert(format == imm_operand, "cannot specify a format"); 1252 return; 1253 } 1254 assert(opnd == pc(), "must put operand where relocs can find it"); 1255 } 1256 #endif // ASSERT 1257 1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) { 1259 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 1260 } 1261 1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) { 1263 if (adr.isxmmindex()) { 1264 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length); 1265 } else { 1266 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 1267 } 1268 } 1269 1270 // Now the Assembler instructions (identical for 32/64 bits) 1271 1272 void Assembler::adcl(Address dst, int32_t imm32) { 1273 InstructionMark im(this); 1274 prefix(dst); 1275 emit_arith_operand(0x81, rdx, dst, imm32); 1276 } 1277 1278 void Assembler::adcl(Address dst, Register src) { 1279 InstructionMark im(this); 1280 prefix(dst, src); 1281 emit_int8(0x11); 1282 emit_operand(src, dst, 0); 1283 } 1284 1285 void Assembler::adcl(Register dst, int32_t imm32) { 1286 prefix(dst); 1287 emit_arith(0x81, 0xD0, dst, imm32); 1288 } 1289 1290 void Assembler::adcl(Register dst, Address src) { 1291 InstructionMark im(this); 1292 prefix(src, dst); 1293 emit_int8(0x13); 1294 emit_operand(dst, src, 0); 1295 } 1296 1297 void Assembler::adcl(Register dst, Register src) { 1298 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1299 emit_arith(0x13, 0xC0, dst, src); 1300 } 1301 1302 void Assembler::addl(Address dst, int32_t imm32) { 1303 InstructionMark im(this); 1304 prefix(dst); 1305 emit_arith_operand(0x81, rax, dst, imm32); 1306 } 1307 1308 void Assembler::addb(Address dst, int imm8) { 1309 InstructionMark im(this); 1310 prefix(dst); 1311 emit_int8((unsigned char)0x80); 1312 emit_operand(rax, dst, 1); 1313 emit_int8(imm8); 1314 } 1315 1316 void Assembler::addw(Register dst, Register src) { 1317 emit_int8(0x66); 1318 (void)prefix_and_encode(dst->encoding(), src->encoding()); 1319 emit_arith(0x03, 0xC0, dst, src); 1320 } 1321 1322 void Assembler::addw(Address dst, int imm16) { 1323 InstructionMark im(this); 1324 emit_int8(0x66); 1325 prefix(dst); 1326 emit_int8((unsigned char)0x81); 1327 emit_operand(rax, dst, 2); 1328 emit_int16(imm16); 1329 } 1330 1331 void Assembler::addl(Address dst, Register src) { 1332 InstructionMark im(this); 1333 prefix(dst, src); 1334 emit_int8(0x01); 1335 emit_operand(src, dst, 0); 1336 } 1337 1338 void Assembler::addl(Register dst, int32_t imm32) { 1339 prefix(dst); 1340 emit_arith(0x81, 0xC0, dst, imm32); 1341 } 1342 1343 void Assembler::addl(Register dst, Address src) { 1344 InstructionMark im(this); 1345 prefix(src, dst); 1346 emit_int8(0x03); 1347 emit_operand(dst, src, 0); 1348 } 1349 1350 void Assembler::addl(Register dst, Register src) { 1351 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1352 emit_arith(0x03, 0xC0, dst, src); 1353 } 1354 1355 void Assembler::addr_nop_4() { 1356 assert(UseAddressNop, "no CPU support"); 1357 // 4 bytes: NOP DWORD PTR [EAX+0] 1358 emit_int32(0x0F, 1359 0x1F, 1360 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc); 1361 0); // 8-bits offset (1 byte) 1362 } 1363 1364 void Assembler::addr_nop_5() { 1365 assert(UseAddressNop, "no CPU support"); 1366 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset 1367 emit_int32(0x0F, 1368 0x1F, 1369 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4); 1370 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc); 1371 emit_int8(0); // 8-bits offset (1 byte) 1372 } 1373 1374 void Assembler::addr_nop_7() { 1375 assert(UseAddressNop, "no CPU support"); 1376 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset 1377 emit_int24(0x0F, 1378 0x1F, 1379 (unsigned char)0x80); 1380 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc); 1381 emit_int32(0); // 32-bits offset (4 bytes) 1382 } 1383 1384 void Assembler::addr_nop_8() { 1385 assert(UseAddressNop, "no CPU support"); 1386 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset 1387 emit_int32(0x0F, 1388 0x1F, 1389 (unsigned char)0x84, 1390 // emit_rm(cbuf, 0x2, EAX_enc, 0x4); 1391 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc); 1392 emit_int32(0); // 32-bits offset (4 bytes) 1393 } 1394 1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 1396 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1397 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1398 attributes.set_rex_vex_w_reverted(); 1399 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 1400 emit_int16(0x58, (0xC0 | encode)); 1401 } 1402 1403 void Assembler::addsd(XMMRegister dst, Address src) { 1404 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1405 InstructionMark im(this); 1406 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1407 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 1408 attributes.set_rex_vex_w_reverted(); 1409 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 1410 emit_int8(0x58); 1411 emit_operand(dst, src, 0); 1412 } 1413 1414 void Assembler::addss(XMMRegister dst, XMMRegister src) { 1415 NOT_LP64(assert(VM_Version::supports_sse(), "")); 1416 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1417 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1418 emit_int16(0x58, (0xC0 | encode)); 1419 } 1420 1421 void Assembler::addss(XMMRegister dst, Address src) { 1422 NOT_LP64(assert(VM_Version::supports_sse(), "")); 1423 InstructionMark im(this); 1424 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1425 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 1426 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1427 emit_int8(0x58); 1428 emit_operand(dst, src, 0); 1429 } 1430 1431 void Assembler::aesdec(XMMRegister dst, Address src) { 1432 assert(VM_Version::supports_aes(), ""); 1433 InstructionMark im(this); 1434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1435 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1436 emit_int8((unsigned char)0xDE); 1437 emit_operand(dst, src, 0); 1438 } 1439 1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) { 1441 assert(VM_Version::supports_aes(), ""); 1442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1443 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1444 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 1445 } 1446 1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1448 assert(VM_Version::supports_avx512_vaes(), ""); 1449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1450 attributes.set_is_evex_instruction(); 1451 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1452 emit_int16((unsigned char)0xDE, (0xC0 | encode)); 1453 } 1454 1455 1456 void Assembler::aesdeclast(XMMRegister dst, Address src) { 1457 assert(VM_Version::supports_aes(), ""); 1458 InstructionMark im(this); 1459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1460 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1461 emit_int8((unsigned char)0xDF); 1462 emit_operand(dst, src, 0); 1463 } 1464 1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) { 1466 assert(VM_Version::supports_aes(), ""); 1467 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1468 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1469 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 1470 } 1471 1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1473 assert(VM_Version::supports_avx512_vaes(), ""); 1474 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1475 attributes.set_is_evex_instruction(); 1476 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1477 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 1478 } 1479 1480 void Assembler::aesenc(XMMRegister dst, Address src) { 1481 assert(VM_Version::supports_aes(), ""); 1482 InstructionMark im(this); 1483 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1484 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1485 emit_int8((unsigned char)0xDC); 1486 emit_operand(dst, src, 0); 1487 } 1488 1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) { 1490 assert(VM_Version::supports_aes(), ""); 1491 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1492 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1493 emit_int16((unsigned char)0xDC, 0xC0 | encode); 1494 } 1495 1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1497 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling"); 1498 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1499 attributes.set_is_evex_instruction(); 1500 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1501 emit_int16((unsigned char)0xDC, (0xC0 | encode)); 1502 } 1503 1504 void Assembler::aesenclast(XMMRegister dst, Address src) { 1505 assert(VM_Version::supports_aes(), ""); 1506 InstructionMark im(this); 1507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1508 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1509 emit_int8((unsigned char)0xDD); 1510 emit_operand(dst, src, 0); 1511 } 1512 1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) { 1514 assert(VM_Version::supports_aes(), ""); 1515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1517 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 1518 } 1519 1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 1521 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling"); 1522 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1523 attributes.set_is_evex_instruction(); 1524 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1525 emit_int16((unsigned char)0xDD, (0xC0 | encode)); 1526 } 1527 1528 void Assembler::andb(Address dst, Register src) { 1529 InstructionMark im(this); 1530 prefix(dst, src, true); 1531 emit_int8(0x20); 1532 emit_operand(src, dst, 0); 1533 } 1534 1535 void Assembler::andw(Register dst, Register src) { 1536 (void)prefix_and_encode(dst->encoding(), src->encoding()); 1537 emit_arith(0x23, 0xC0, dst, src); 1538 } 1539 1540 void Assembler::andl(Address dst, int32_t imm32) { 1541 InstructionMark im(this); 1542 prefix(dst); 1543 emit_arith_operand(0x81, as_Register(4), dst, imm32); 1544 } 1545 1546 void Assembler::andl(Register dst, int32_t imm32) { 1547 prefix(dst); 1548 emit_arith(0x81, 0xE0, dst, imm32); 1549 } 1550 1551 void Assembler::andl(Address dst, Register src) { 1552 InstructionMark im(this); 1553 prefix(dst, src); 1554 emit_int8(0x21); 1555 emit_operand(src, dst, 0); 1556 } 1557 1558 void Assembler::andl(Register dst, Address src) { 1559 InstructionMark im(this); 1560 prefix(src, dst); 1561 emit_int8(0x23); 1562 emit_operand(dst, src, 0); 1563 } 1564 1565 void Assembler::andl(Register dst, Register src) { 1566 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1567 emit_arith(0x23, 0xC0, dst, src); 1568 } 1569 1570 void Assembler::andnl(Register dst, Register src1, Register src2) { 1571 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1572 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1573 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1574 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 1575 } 1576 1577 void Assembler::andnl(Register dst, Register src1, Address src2) { 1578 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1579 InstructionMark im(this); 1580 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1581 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1582 emit_int8((unsigned char)0xF2); 1583 emit_operand(dst, src2, 0); 1584 } 1585 1586 void Assembler::bsfl(Register dst, Register src) { 1587 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 1588 emit_int24(0x0F, 1589 (unsigned char)0xBC, 1590 0xC0 | encode); 1591 } 1592 1593 void Assembler::bsrl(Register dst, Register src) { 1594 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 1595 emit_int24(0x0F, 1596 (unsigned char)0xBD, 1597 0xC0 | encode); 1598 } 1599 1600 void Assembler::bswapl(Register reg) { // bswap 1601 int encode = prefix_and_encode(reg->encoding()); 1602 emit_int16(0x0F, (0xC8 | encode)); 1603 } 1604 1605 void Assembler::blsil(Register dst, Register src) { 1606 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1607 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1608 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1609 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 1610 } 1611 1612 void Assembler::blsil(Register dst, Address src) { 1613 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1614 InstructionMark im(this); 1615 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1616 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1617 emit_int8((unsigned char)0xF3); 1618 emit_operand(rbx, src, 0); 1619 } 1620 1621 void Assembler::blsmskl(Register dst, Register src) { 1622 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1623 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1624 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1625 emit_int16((unsigned char)0xF3, 1626 0xC0 | encode); 1627 } 1628 1629 void Assembler::blsmskl(Register dst, Address src) { 1630 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1631 InstructionMark im(this); 1632 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1633 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1634 emit_int8((unsigned char)0xF3); 1635 emit_operand(rdx, src, 0); 1636 } 1637 1638 void Assembler::blsrl(Register dst, Register src) { 1639 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1640 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1641 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1642 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 1643 } 1644 1645 void Assembler::blsrl(Register dst, Address src) { 1646 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 1647 InstructionMark im(this); 1648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 1649 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 1650 emit_int8((unsigned char)0xF3); 1651 emit_operand(rcx, src, 0); 1652 } 1653 1654 void Assembler::call(Label& L, relocInfo::relocType rtype) { 1655 // suspect disp32 is always good 1656 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand); 1657 1658 if (L.is_bound()) { 1659 const int long_size = 5; 1660 int offs = (int)( target(L) - pc() ); 1661 assert(offs <= 0, "assembler error"); 1662 InstructionMark im(this); 1663 // 1110 1000 #32-bit disp 1664 emit_int8((unsigned char)0xE8); 1665 emit_data(offs - long_size, rtype, operand); 1666 } else { 1667 InstructionMark im(this); 1668 // 1110 1000 #32-bit disp 1669 L.add_patch_at(code(), locator()); 1670 1671 emit_int8((unsigned char)0xE8); 1672 emit_data(int(0), rtype, operand); 1673 } 1674 } 1675 1676 void Assembler::call(Register dst) { 1677 int encode = prefix_and_encode(dst->encoding()); 1678 emit_int16((unsigned char)0xFF, (0xD0 | encode)); 1679 } 1680 1681 1682 void Assembler::call(Address adr) { 1683 InstructionMark im(this); 1684 prefix(adr); 1685 emit_int8((unsigned char)0xFF); 1686 emit_operand(rdx, adr, 0); 1687 } 1688 1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) { 1690 InstructionMark im(this); 1691 emit_int8((unsigned char)0xE8); 1692 intptr_t disp = entry - (pc() + sizeof(int32_t)); 1693 // Entry is NULL in case of a scratch emit. 1694 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp); 1695 // Technically, should use call32_operand, but this format is 1696 // implied by the fact that we're emitting a call instruction. 1697 1698 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand); 1699 emit_data((int) disp, rspec, operand); 1700 } 1701 1702 void Assembler::cdql() { 1703 emit_int8((unsigned char)0x99); 1704 } 1705 1706 void Assembler::cld() { 1707 emit_int8((unsigned char)0xFC); 1708 } 1709 1710 void Assembler::cmovl(Condition cc, Register dst, Register src) { 1711 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction")); 1712 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 1713 emit_int24(0x0F, 1714 0x40 | cc, 1715 0xC0 | encode); 1716 } 1717 1718 1719 void Assembler::cmovl(Condition cc, Register dst, Address src) { 1720 InstructionMark im(this); 1721 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction")); 1722 prefix(src, dst); 1723 emit_int16(0x0F, (0x40 | cc)); 1724 emit_operand(dst, src, 0); 1725 } 1726 1727 void Assembler::cmpb(Address dst, int imm8) { 1728 InstructionMark im(this); 1729 prefix(dst); 1730 emit_int8((unsigned char)0x80); 1731 emit_operand(rdi, dst, 1); 1732 emit_int8(imm8); 1733 } 1734 1735 void Assembler::cmpl(Address dst, int32_t imm32) { 1736 InstructionMark im(this); 1737 prefix(dst); 1738 emit_arith_operand(0x81, as_Register(7), dst, imm32); 1739 } 1740 1741 void Assembler::cmpl(Register dst, int32_t imm32) { 1742 prefix(dst); 1743 emit_arith(0x81, 0xF8, dst, imm32); 1744 } 1745 1746 void Assembler::cmpl(Register dst, Register src) { 1747 (void) prefix_and_encode(dst->encoding(), src->encoding()); 1748 emit_arith(0x3B, 0xC0, dst, src); 1749 } 1750 1751 void Assembler::cmpl(Register dst, Address src) { 1752 InstructionMark im(this); 1753 prefix(src, dst); 1754 emit_int8(0x3B); 1755 emit_operand(dst, src, 0); 1756 } 1757 1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) { 1759 InstructionMark im(this); 1760 prefix(dst); 1761 emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32); 1762 } 1763 1764 void Assembler::cmpw(Address dst, int imm16) { 1765 InstructionMark im(this); 1766 emit_int8(0x66); 1767 prefix(dst); 1768 emit_int8((unsigned char)0x81); 1769 emit_operand(rdi, dst, 2); 1770 emit_int16(imm16); 1771 } 1772 1773 // The 32-bit cmpxchg compares the value at adr with the contents of rax, 1774 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,. 1775 // The ZF is set if the compared values were equal, and cleared otherwise. 1776 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg 1777 InstructionMark im(this); 1778 prefix(adr, reg); 1779 emit_int16(0x0F, (unsigned char)0xB1); 1780 emit_operand(reg, adr, 0); 1781 } 1782 1783 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg 1784 InstructionMark im(this); 1785 size_prefix(); 1786 prefix(adr, reg); 1787 emit_int16(0x0F, (unsigned char)0xB1); 1788 emit_operand(reg, adr, 0); 1789 } 1790 1791 // The 8-bit cmpxchg compares the value at adr with the contents of rax, 1792 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,. 1793 // The ZF is set if the compared values were equal, and cleared otherwise. 1794 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg 1795 InstructionMark im(this); 1796 prefix(adr, reg, true); 1797 emit_int16(0x0F, (unsigned char)0xB0); 1798 emit_operand(reg, adr, 0); 1799 } 1800 1801 void Assembler::comisd(XMMRegister dst, Address src) { 1802 // NOTE: dbx seems to decode this as comiss even though the 1803 // 0x66 is there. Strangely ucomisd comes out correct 1804 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1805 InstructionMark im(this); 1806 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);; 1807 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 1808 attributes.set_rex_vex_w_reverted(); 1809 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 1810 emit_int8(0x2F); 1811 emit_operand(dst, src, 0); 1812 } 1813 1814 void Assembler::comisd(XMMRegister dst, XMMRegister src) { 1815 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1816 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1817 attributes.set_rex_vex_w_reverted(); 1818 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 1819 emit_int16(0x2F, (0xC0 | encode)); 1820 } 1821 1822 void Assembler::comiss(XMMRegister dst, Address src) { 1823 NOT_LP64(assert(VM_Version::supports_sse(), "")); 1824 InstructionMark im(this); 1825 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1826 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 1827 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1828 emit_int8(0x2F); 1829 emit_operand(dst, src, 0); 1830 } 1831 1832 void Assembler::comiss(XMMRegister dst, XMMRegister src) { 1833 NOT_LP64(assert(VM_Version::supports_sse(), "")); 1834 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 1835 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1836 emit_int16(0x2F, (0xC0 | encode)); 1837 } 1838 1839 void Assembler::cpuid() { 1840 emit_int16(0x0F, (unsigned char)0xA2); 1841 } 1842 1843 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented 1844 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v 1845 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. - 1846 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. - 1847 // 1848 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v 1849 // 1850 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v 1851 // 1852 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v 1853 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) { 1854 assert(VM_Version::supports_sse4_2(), ""); 1855 int8_t w = 0x01; 1856 Prefix p = Prefix_EMPTY; 1857 1858 emit_int8((unsigned char)0xF2); 1859 switch (sizeInBytes) { 1860 case 1: 1861 w = 0; 1862 break; 1863 case 2: 1864 case 4: 1865 break; 1866 LP64_ONLY(case 8:) 1867 // This instruction is not valid in 32 bits 1868 // Note: 1869 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf 1870 // 1871 // Page B - 72 Vol. 2C says 1872 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2 1873 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m 1874 // F0!!! 1875 // while 3 - 208 Vol. 2A 1876 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64. 1877 // 1878 // the 0 on a last bit is reserved for a different flavor of this instruction : 1879 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8. 1880 p = REX_W; 1881 break; 1882 default: 1883 assert(0, "Unsupported value for a sizeInBytes argument"); 1884 break; 1885 } 1886 LP64_ONLY(prefix(crc, v, p);) 1887 emit_int32(0x0F, 1888 0x38, 1889 0xF0 | w, 1890 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7)); 1891 } 1892 1893 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) { 1894 assert(VM_Version::supports_sse4_2(), ""); 1895 InstructionMark im(this); 1896 int8_t w = 0x01; 1897 Prefix p = Prefix_EMPTY; 1898 1899 emit_int8((int8_t)0xF2); 1900 switch (sizeInBytes) { 1901 case 1: 1902 w = 0; 1903 break; 1904 case 2: 1905 case 4: 1906 break; 1907 LP64_ONLY(case 8:) 1908 // This instruction is not valid in 32 bits 1909 p = REX_W; 1910 break; 1911 default: 1912 assert(0, "Unsupported value for a sizeInBytes argument"); 1913 break; 1914 } 1915 LP64_ONLY(prefix(crc, adr, p);) 1916 emit_int24(0x0F, 0x38, (0xF0 | w)); 1917 emit_operand(crc, adr, 0); 1918 } 1919 1920 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) { 1921 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1922 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1923 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1924 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 1925 } 1926 1927 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) { 1928 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 1929 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1930 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 1931 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 1932 } 1933 1934 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 1935 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 1936 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true); 1937 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 1938 emit_int24(0x1D, (0xC0 | encode), imm8); 1939 } 1940 1941 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) { 1942 assert(VM_Version::supports_evex(), ""); 1943 InstructionMark im(this); 1944 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true); 1945 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit); 1946 attributes.reset_is_clear_context(); 1947 attributes.set_embedded_opmask_register_specifier(mask); 1948 attributes.set_is_evex_instruction(); 1949 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 1950 emit_int8(0x1D); 1951 emit_operand(src, dst, 1); 1952 emit_int8(imm8); 1953 } 1954 1955 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) { 1956 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 1957 InstructionMark im(this); 1958 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true); 1959 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 1960 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 1961 emit_int8(0x1D); 1962 emit_operand(src, dst, 1); 1963 emit_int8(imm8); 1964 } 1965 1966 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) { 1967 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 1968 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true); 1969 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1970 emit_int16(0x13, (0xC0 | encode)); 1971 } 1972 1973 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) { 1974 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), ""); 1975 InstructionMark im(this); 1976 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true); 1977 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 1978 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 1979 emit_int8(0x13); 1980 emit_operand(dst, src, 0); 1981 } 1982 1983 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) { 1984 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1985 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1986 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1987 emit_int16(0x5B, (0xC0 | encode)); 1988 } 1989 1990 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) { 1991 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 1992 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 1993 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 1994 emit_int16(0x5B, (0xC0 | encode)); 1995 } 1996 1997 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 1998 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 1999 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2000 attributes.set_rex_vex_w_reverted(); 2001 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2002 emit_int16(0x5A, (0xC0 | encode)); 2003 } 2004 2005 void Assembler::cvtsd2ss(XMMRegister dst, Address src) { 2006 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2007 InstructionMark im(this); 2008 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 2010 attributes.set_rex_vex_w_reverted(); 2011 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2012 emit_int8(0x5A); 2013 emit_operand(dst, src, 0); 2014 } 2015 2016 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) { 2017 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2018 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2019 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2020 emit_int16(0x2A, (0xC0 | encode)); 2021 } 2022 2023 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) { 2024 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2025 InstructionMark im(this); 2026 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2027 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2028 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2029 emit_int8(0x2A); 2030 emit_operand(dst, src, 0); 2031 } 2032 2033 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) { 2034 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2035 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2036 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2037 emit_int16(0x2A, (0xC0 | encode)); 2038 } 2039 2040 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) { 2041 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2042 InstructionMark im(this); 2043 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2044 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2045 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2046 emit_int8(0x2A); 2047 emit_operand(dst, src, 0); 2048 } 2049 2050 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) { 2051 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2052 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2053 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2054 emit_int16(0x2A, (0xC0 | encode)); 2055 } 2056 2057 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 2058 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2059 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2060 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2061 emit_int16(0x5A, (0xC0 | encode)); 2062 } 2063 2064 void Assembler::cvtss2sd(XMMRegister dst, Address src) { 2065 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2066 InstructionMark im(this); 2067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2068 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2069 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2070 emit_int8(0x5A); 2071 emit_operand(dst, src, 0); 2072 } 2073 2074 2075 void Assembler::cvttsd2sil(Register dst, XMMRegister src) { 2076 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2077 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2078 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2079 emit_int16(0x2C, (0xC0 | encode)); 2080 } 2081 2082 void Assembler::cvtss2sil(Register dst, XMMRegister src) { 2083 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2084 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2085 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2086 emit_int16(0x2D, (0xC0 | encode)); 2087 } 2088 2089 void Assembler::cvttss2sil(Register dst, XMMRegister src) { 2090 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2091 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2092 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2093 emit_int16(0x2C, (0xC0 | encode)); 2094 } 2095 2096 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) { 2097 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2098 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2099 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2100 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2101 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2102 } 2103 2104 void Assembler::pabsb(XMMRegister dst, XMMRegister src) { 2105 assert(VM_Version::supports_ssse3(), ""); 2106 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2107 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2108 emit_int16(0x1C, (0xC0 | encode)); 2109 } 2110 2111 void Assembler::pabsw(XMMRegister dst, XMMRegister src) { 2112 assert(VM_Version::supports_ssse3(), ""); 2113 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2114 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2115 emit_int16(0x1D, (0xC0 | encode)); 2116 } 2117 2118 void Assembler::pabsd(XMMRegister dst, XMMRegister src) { 2119 assert(VM_Version::supports_ssse3(), ""); 2120 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2121 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2122 emit_int16(0x1E, (0xC0 | encode)); 2123 } 2124 2125 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) { 2126 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 2127 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 2128 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported"); 2129 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2130 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2131 emit_int16(0x1C, (0xC0 | encode)); 2132 } 2133 2134 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) { 2135 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 2136 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 2137 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, ""); 2138 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 2139 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2140 emit_int16(0x1D, (0xC0 | encode)); 2141 } 2142 2143 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) { 2144 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 2145 vector_len == AVX_256bit? VM_Version::supports_avx2() : 2146 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, ""); 2147 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2148 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2149 emit_int16(0x1E, (0xC0 | encode)); 2150 } 2151 2152 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) { 2153 assert(UseAVX > 2, ""); 2154 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2155 attributes.set_is_evex_instruction(); 2156 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 2157 emit_int16(0x1F, (0xC0 | encode)); 2158 } 2159 2160 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) { 2161 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2162 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2164 emit_int16(0x5A, (0xC0 | encode)); 2165 } 2166 2167 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2168 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2169 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2170 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2171 attributes.set_rex_vex_w_reverted(); 2172 emit_int16(0x5A, (0xC0 | encode)); 2173 } 2174 2175 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2176 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2177 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2178 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2179 emit_int16(0x5B, (0xC0 | encode)); 2180 } 2181 2182 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2183 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2184 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2185 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2186 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2187 } 2188 2189 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) { 2190 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 2191 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2192 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2193 emit_int16(0x5B, (0xC0 | encode)); 2194 } 2195 2196 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2197 assert(VM_Version::supports_avx512dq(), ""); 2198 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2199 attributes.set_is_evex_instruction(); 2200 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2201 emit_int16(0x7A, (0xC0 | encode)); 2202 } 2203 2204 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2205 assert(VM_Version::supports_avx512dq(), ""); 2206 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2207 attributes.set_is_evex_instruction(); 2208 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2209 emit_int16(0x7B, (0xC0 | encode)); 2210 } 2211 2212 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) { 2213 assert(VM_Version::supports_avx512dq(), ""); 2214 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2215 attributes.set_is_evex_instruction(); 2216 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2217 emit_int16(0x5B, (0xC0 | encode)); 2218 } 2219 2220 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) { 2221 assert(VM_Version::supports_avx512dq(), ""); 2222 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2223 attributes.set_is_evex_instruction(); 2224 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2225 emit_int16(0x7A, (0xC0 | encode)); 2226 } 2227 2228 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) { 2229 assert(VM_Version::supports_avx512dq(), ""); 2230 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2231 attributes.set_is_evex_instruction(); 2232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2233 emit_int16((unsigned char)0xE6, (0xC0 | encode)); 2234 } 2235 2236 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) { 2237 assert(VM_Version::supports_avx512bw(), ""); 2238 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2239 attributes.set_is_evex_instruction(); 2240 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2241 emit_int16(0x30, (0xC0 | encode)); 2242 } 2243 2244 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) { 2245 assert(UseAVX > 2, ""); 2246 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2247 attributes.set_is_evex_instruction(); 2248 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2249 emit_int16(0x33, (0xC0 | encode)); 2250 } 2251 2252 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) { 2253 assert(UseAVX > 2, ""); 2254 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2255 attributes.set_is_evex_instruction(); 2256 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2257 emit_int16(0x31, (0xC0 | encode)); 2258 } 2259 2260 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) { 2261 assert(UseAVX > 2, ""); 2262 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2263 attributes.set_is_evex_instruction(); 2264 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2265 emit_int16(0x35, (0xC0 | encode)); 2266 } 2267 2268 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) { 2269 assert(UseAVX > 2, ""); 2270 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2271 attributes.set_is_evex_instruction(); 2272 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2273 emit_int16(0x32, (0xC0 | encode)); 2274 } 2275 2276 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) { 2277 assert(UseAVX > 2, ""); 2278 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2279 attributes.set_is_evex_instruction(); 2280 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2281 emit_int16(0x34, (0xC0 | encode)); 2282 } 2283 2284 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) { 2285 assert(UseAVX > 2, ""); 2286 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2287 attributes.set_is_evex_instruction(); 2288 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 2289 emit_int16(0x25, (0xC0 | encode)); 2290 } 2291 2292 void Assembler::decl(Address dst) { 2293 // Don't use it directly. Use MacroAssembler::decrement() instead. 2294 InstructionMark im(this); 2295 prefix(dst); 2296 emit_int8((unsigned char)0xFF); 2297 emit_operand(rcx, dst, 0); 2298 } 2299 2300 void Assembler::divsd(XMMRegister dst, Address src) { 2301 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2302 InstructionMark im(this); 2303 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2304 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 2305 attributes.set_rex_vex_w_reverted(); 2306 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2307 emit_int8(0x5E); 2308 emit_operand(dst, src, 0); 2309 } 2310 2311 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 2312 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2313 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2314 attributes.set_rex_vex_w_reverted(); 2315 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2316 emit_int16(0x5E, (0xC0 | encode)); 2317 } 2318 2319 void Assembler::divss(XMMRegister dst, Address src) { 2320 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2321 InstructionMark im(this); 2322 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2323 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 2324 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2325 emit_int8(0x5E); 2326 emit_operand(dst, src, 0); 2327 } 2328 2329 void Assembler::divss(XMMRegister dst, XMMRegister src) { 2330 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2331 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 2332 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 2333 emit_int16(0x5E, (0xC0 | encode)); 2334 } 2335 2336 void Assembler::hlt() { 2337 emit_int8((unsigned char)0xF4); 2338 } 2339 2340 void Assembler::idivl(Register src) { 2341 int encode = prefix_and_encode(src->encoding()); 2342 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 2343 } 2344 2345 void Assembler::divl(Register src) { // Unsigned 2346 int encode = prefix_and_encode(src->encoding()); 2347 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 2348 } 2349 2350 void Assembler::imull(Register src) { 2351 int encode = prefix_and_encode(src->encoding()); 2352 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 2353 } 2354 2355 void Assembler::imull(Register dst, Register src) { 2356 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 2357 emit_int24(0x0F, 2358 (unsigned char)0xAF, 2359 (0xC0 | encode)); 2360 } 2361 2362 void Assembler::imull(Register dst, Address src, int32_t value) { 2363 InstructionMark im(this); 2364 prefix(src, dst); 2365 if (is8bit(value)) { 2366 emit_int8((unsigned char)0x6B); 2367 emit_operand(dst, src, 1); 2368 emit_int8(value); 2369 } else { 2370 emit_int8((unsigned char)0x69); 2371 emit_operand(dst, src, 4); 2372 emit_int32(value); 2373 } 2374 } 2375 2376 void Assembler::imull(Register dst, Register src, int value) { 2377 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 2378 if (is8bit(value)) { 2379 emit_int24(0x6B, (0xC0 | encode), value & 0xFF); 2380 } else { 2381 emit_int16(0x69, (0xC0 | encode)); 2382 emit_int32(value); 2383 } 2384 } 2385 2386 void Assembler::imull(Register dst, Address src) { 2387 InstructionMark im(this); 2388 prefix(src, dst); 2389 emit_int16(0x0F, (unsigned char)0xAF); 2390 emit_operand(dst, src, 0); 2391 } 2392 2393 2394 void Assembler::incl(Address dst) { 2395 // Don't use it directly. Use MacroAssembler::increment() instead. 2396 InstructionMark im(this); 2397 prefix(dst); 2398 emit_int8((unsigned char)0xFF); 2399 emit_operand(rax, dst, 0); 2400 } 2401 2402 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) { 2403 InstructionMark im(this); 2404 assert((0 <= cc) && (cc < 16), "illegal cc"); 2405 if (L.is_bound()) { 2406 address dst = target(L); 2407 assert(dst != NULL, "jcc most probably wrong"); 2408 2409 const int short_size = 2; 2410 const int long_size = 6; 2411 intptr_t offs = (intptr_t)dst - (intptr_t)pc(); 2412 if (maybe_short && is8bit(offs - short_size)) { 2413 // 0111 tttn #8-bit disp 2414 emit_int16(0x70 | cc, (offs - short_size) & 0xFF); 2415 } else { 2416 // 0000 1111 1000 tttn #32-bit disp 2417 assert(is_simm32(offs - long_size), 2418 "must be 32bit offset (call4)"); 2419 emit_int16(0x0F, (0x80 | cc)); 2420 emit_int32(offs - long_size); 2421 } 2422 } else { 2423 // Note: could eliminate cond. jumps to this jump if condition 2424 // is the same however, seems to be rather unlikely case. 2425 // Note: use jccb() if label to be bound is very close to get 2426 // an 8-bit displacement 2427 L.add_patch_at(code(), locator()); 2428 emit_int16(0x0F, (0x80 | cc)); 2429 emit_int32(0); 2430 } 2431 } 2432 2433 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) { 2434 if (L.is_bound()) { 2435 const int short_size = 2; 2436 address entry = target(L); 2437 #ifdef ASSERT 2438 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size); 2439 intptr_t delta = short_branch_delta(); 2440 if (delta != 0) { 2441 dist += (dist < 0 ? (-delta) :delta); 2442 } 2443 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line); 2444 #endif 2445 intptr_t offs = (intptr_t)entry - (intptr_t)pc(); 2446 // 0111 tttn #8-bit disp 2447 emit_int16(0x70 | cc, (offs - short_size) & 0xFF); 2448 } else { 2449 InstructionMark im(this); 2450 L.add_patch_at(code(), locator(), file, line); 2451 emit_int16(0x70 | cc, 0); 2452 } 2453 } 2454 2455 void Assembler::jmp(Address adr) { 2456 InstructionMark im(this); 2457 prefix(adr); 2458 emit_int8((unsigned char)0xFF); 2459 emit_operand(rsp, adr, 0); 2460 } 2461 2462 void Assembler::jmp(Label& L, bool maybe_short) { 2463 if (L.is_bound()) { 2464 address entry = target(L); 2465 assert(entry != NULL, "jmp most probably wrong"); 2466 InstructionMark im(this); 2467 const int short_size = 2; 2468 const int long_size = 5; 2469 intptr_t offs = entry - pc(); 2470 if (maybe_short && is8bit(offs - short_size)) { 2471 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF)); 2472 } else { 2473 emit_int8((unsigned char)0xE9); 2474 emit_int32(offs - long_size); 2475 } 2476 } else { 2477 // By default, forward jumps are always 32-bit displacements, since 2478 // we can't yet know where the label will be bound. If you're sure that 2479 // the forward jump will not run beyond 256 bytes, use jmpb to 2480 // force an 8-bit displacement. 2481 InstructionMark im(this); 2482 L.add_patch_at(code(), locator()); 2483 emit_int8((unsigned char)0xE9); 2484 emit_int32(0); 2485 } 2486 } 2487 2488 void Assembler::jmp(Register entry) { 2489 int encode = prefix_and_encode(entry->encoding()); 2490 emit_int16((unsigned char)0xFF, (0xE0 | encode)); 2491 } 2492 2493 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) { 2494 InstructionMark im(this); 2495 emit_int8((unsigned char)0xE9); 2496 assert(dest != NULL, "must have a target"); 2497 intptr_t disp = dest - (pc() + sizeof(int32_t)); 2498 assert(is_simm32(disp), "must be 32bit offset (jmp)"); 2499 emit_data(disp, rspec, call32_operand); 2500 } 2501 2502 void Assembler::jmpb_0(Label& L, const char* file, int line) { 2503 if (L.is_bound()) { 2504 const int short_size = 2; 2505 address entry = target(L); 2506 assert(entry != NULL, "jmp most probably wrong"); 2507 #ifdef ASSERT 2508 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size); 2509 intptr_t delta = short_branch_delta(); 2510 if (delta != 0) { 2511 dist += (dist < 0 ? (-delta) :delta); 2512 } 2513 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line); 2514 #endif 2515 intptr_t offs = entry - pc(); 2516 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF); 2517 } else { 2518 InstructionMark im(this); 2519 L.add_patch_at(code(), locator(), file, line); 2520 emit_int16((unsigned char)0xEB, 0); 2521 } 2522 } 2523 2524 void Assembler::ldmxcsr( Address src) { 2525 if (UseAVX > 0 ) { 2526 InstructionMark im(this); 2527 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2528 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2529 emit_int8((unsigned char)0xAE); 2530 emit_operand(as_Register(2), src, 0); 2531 } else { 2532 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2533 InstructionMark im(this); 2534 prefix(src); 2535 emit_int16(0x0F, (unsigned char)0xAE); 2536 emit_operand(as_Register(2), src, 0); 2537 } 2538 } 2539 2540 void Assembler::leal(Register dst, Address src) { 2541 InstructionMark im(this); 2542 prefix(src, dst); 2543 emit_int8((unsigned char)0x8D); 2544 emit_operand(dst, src, 0); 2545 } 2546 2547 void Assembler::lfence() { 2548 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8); 2549 } 2550 2551 void Assembler::lock() { 2552 emit_int8((unsigned char)0xF0); 2553 } 2554 2555 void Assembler::size_prefix() { 2556 emit_int8(0x66); 2557 } 2558 2559 void Assembler::lzcntl(Register dst, Register src) { 2560 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2561 emit_int8((unsigned char)0xF3); 2562 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 2563 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode)); 2564 } 2565 2566 void Assembler::lzcntl(Register dst, Address src) { 2567 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 2568 InstructionMark im(this); 2569 emit_int8((unsigned char)0xF3); 2570 prefix(src, dst); 2571 emit_int16(0x0F, (unsigned char)0xBD); 2572 emit_operand(dst, src, 0); 2573 } 2574 2575 // Emit mfence instruction 2576 void Assembler::mfence() { 2577 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");) 2578 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0); 2579 } 2580 2581 // Emit sfence instruction 2582 void Assembler::sfence() { 2583 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");) 2584 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8); 2585 } 2586 2587 void Assembler::mov(Register dst, Register src) { 2588 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2589 } 2590 2591 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 2592 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 2593 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2594 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2595 attributes.set_rex_vex_w_reverted(); 2596 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2597 emit_int16(0x28, (0xC0 | encode)); 2598 } 2599 2600 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 2601 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2602 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2603 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2604 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2605 emit_int16(0x28, (0xC0 | encode)); 2606 } 2607 2608 void Assembler::movlhps(XMMRegister dst, XMMRegister src) { 2609 NOT_LP64(assert(VM_Version::supports_sse(), "")); 2610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2611 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2612 emit_int16(0x16, (0xC0 | encode)); 2613 } 2614 2615 void Assembler::movb(Register dst, Address src) { 2616 NOT_LP64(assert(dst->has_byte_register(), "must have byte register")); 2617 InstructionMark im(this); 2618 prefix(src, dst, true); 2619 emit_int8((unsigned char)0x8A); 2620 emit_operand(dst, src, 0); 2621 } 2622 2623 void Assembler::movddup(XMMRegister dst, XMMRegister src) { 2624 NOT_LP64(assert(VM_Version::supports_sse3(), "")); 2625 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 2626 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2627 attributes.set_rex_vex_w_reverted(); 2628 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2629 emit_int16(0x12, 0xC0 | encode); 2630 } 2631 2632 void Assembler::movddup(XMMRegister dst, Address src) { 2633 NOT_LP64(assert(VM_Version::supports_sse3(), "")); 2634 InstructionMark im(this); 2635 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2636 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit); 2637 attributes.set_rex_vex_w_reverted(); 2638 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2639 emit_int8(0x12); 2640 emit_operand(dst, src, 0); 2641 } 2642 2643 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) { 2644 assert(VM_Version::supports_avx(), ""); 2645 InstructionMark im(this); 2646 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 2647 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit); 2648 attributes.set_rex_vex_w_reverted(); 2649 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2650 emit_int8(0x12); 2651 emit_operand(dst, src, 0); 2652 } 2653 2654 void Assembler::kmovbl(KRegister dst, KRegister src) { 2655 assert(VM_Version::supports_avx512dq(), ""); 2656 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2657 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2658 emit_int16((unsigned char)0x90, (0xC0 | encode)); 2659 } 2660 2661 void Assembler::kmovbl(KRegister dst, Register src) { 2662 assert(VM_Version::supports_avx512dq(), ""); 2663 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2664 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2665 emit_int16((unsigned char)0x92, (0xC0 | encode)); 2666 } 2667 2668 void Assembler::kmovbl(Register dst, KRegister src) { 2669 assert(VM_Version::supports_avx512dq(), ""); 2670 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2671 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2672 emit_int16((unsigned char)0x93, (0xC0 | encode)); 2673 } 2674 2675 void Assembler::kmovwl(KRegister dst, Register src) { 2676 assert(VM_Version::supports_evex(), ""); 2677 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2678 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2679 emit_int16((unsigned char)0x92, (0xC0 | encode)); 2680 } 2681 2682 void Assembler::kmovwl(Register dst, KRegister src) { 2683 assert(VM_Version::supports_evex(), ""); 2684 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2685 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2686 emit_int16((unsigned char)0x93, (0xC0 | encode)); 2687 } 2688 2689 void Assembler::kmovwl(KRegister dst, Address src) { 2690 assert(VM_Version::supports_evex(), ""); 2691 InstructionMark im(this); 2692 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2693 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2694 emit_int8((unsigned char)0x90); 2695 emit_operand(dst, src, 0); 2696 } 2697 2698 void Assembler::kmovwl(Address dst, KRegister src) { 2699 assert(VM_Version::supports_evex(), ""); 2700 InstructionMark im(this); 2701 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2702 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2703 emit_int8((unsigned char)0x91); 2704 emit_operand(src, dst, 0); 2705 } 2706 2707 void Assembler::kmovwl(KRegister dst, KRegister src) { 2708 assert(VM_Version::supports_evex(), ""); 2709 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2710 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2711 emit_int16((unsigned char)0x90, (0xC0 | encode)); 2712 } 2713 2714 void Assembler::kmovdl(KRegister dst, Register src) { 2715 assert(VM_Version::supports_avx512bw(), ""); 2716 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2717 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2718 emit_int16((unsigned char)0x92, (0xC0 | encode)); 2719 } 2720 2721 void Assembler::kmovdl(Register dst, KRegister src) { 2722 assert(VM_Version::supports_avx512bw(), ""); 2723 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2724 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2725 emit_int16((unsigned char)0x93, (0xC0 | encode)); 2726 } 2727 2728 void Assembler::kmovql(KRegister dst, KRegister src) { 2729 assert(VM_Version::supports_avx512bw(), ""); 2730 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2731 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2732 emit_int16((unsigned char)0x90, (0xC0 | encode)); 2733 } 2734 2735 void Assembler::kmovql(KRegister dst, Address src) { 2736 assert(VM_Version::supports_avx512bw(), ""); 2737 InstructionMark im(this); 2738 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2739 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2740 emit_int8((unsigned char)0x90); 2741 emit_operand(dst, src, 0); 2742 } 2743 2744 void Assembler::kmovql(Address dst, KRegister src) { 2745 assert(VM_Version::supports_avx512bw(), ""); 2746 InstructionMark im(this); 2747 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2748 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2749 emit_int8((unsigned char)0x91); 2750 emit_operand(src, dst, 0); 2751 } 2752 2753 void Assembler::kmovql(KRegister dst, Register src) { 2754 assert(VM_Version::supports_avx512bw(), ""); 2755 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2756 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2757 emit_int16((unsigned char)0x92, (0xC0 | encode)); 2758 } 2759 2760 void Assembler::kmovql(Register dst, KRegister src) { 2761 assert(VM_Version::supports_avx512bw(), ""); 2762 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2763 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 2764 emit_int16((unsigned char)0x93, (0xC0 | encode)); 2765 } 2766 2767 void Assembler::knotwl(KRegister dst, KRegister src) { 2768 assert(VM_Version::supports_evex(), ""); 2769 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2770 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2771 emit_int16(0x44, (0xC0 | encode)); 2772 } 2773 2774 void Assembler::knotbl(KRegister dst, KRegister src) { 2775 assert(VM_Version::supports_evex(), ""); 2776 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2777 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2778 emit_int16(0x44, (0xC0 | encode)); 2779 } 2780 2781 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) { 2782 assert(VM_Version::supports_avx512dq(), ""); 2783 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2784 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2785 emit_int16(0x45, (0xC0 | encode)); 2786 } 2787 2788 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) { 2789 assert(VM_Version::supports_evex(), ""); 2790 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2791 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2792 emit_int16(0x45, (0xC0 | encode)); 2793 } 2794 2795 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) { 2796 assert(VM_Version::supports_avx512bw(), ""); 2797 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2798 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2799 emit_int16(0x45, (0xC0 | encode)); 2800 } 2801 2802 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) { 2803 assert(VM_Version::supports_avx512bw(), ""); 2804 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2805 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2806 emit_int16(0x45, (0xC0 | encode)); 2807 } 2808 2809 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) { 2810 assert(VM_Version::supports_avx512dq(), ""); 2811 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2812 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2813 emit_int16(0x47, (0xC0 | encode)); 2814 } 2815 2816 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) { 2817 assert(VM_Version::supports_evex(), ""); 2818 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2819 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2820 emit_int16(0x47, (0xC0 | encode)); 2821 } 2822 2823 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) { 2824 assert(VM_Version::supports_avx512bw(), ""); 2825 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2826 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2827 emit_int16(0x47, (0xC0 | encode)); 2828 } 2829 2830 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) { 2831 assert(VM_Version::supports_avx512bw(), ""); 2832 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2833 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2834 emit_int16(0x47, (0xC0 | encode)); 2835 } 2836 2837 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) { 2838 assert(VM_Version::supports_avx512dq(), ""); 2839 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2840 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2841 emit_int16(0x41, (0xC0 | encode)); 2842 } 2843 2844 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) { 2845 assert(VM_Version::supports_evex(), ""); 2846 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2847 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2848 emit_int16(0x41, (0xC0 | encode)); 2849 } 2850 2851 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) { 2852 assert(VM_Version::supports_avx512bw(), ""); 2853 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2854 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2855 emit_int16(0x41, (0xC0 | encode)); 2856 } 2857 2858 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) { 2859 assert(VM_Version::supports_avx512bw(), ""); 2860 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2861 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2862 emit_int16(0x41, (0xC0 | encode)); 2863 } 2864 2865 void Assembler::knotdl(KRegister dst, KRegister src) { 2866 assert(VM_Version::supports_avx512bw(), ""); 2867 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2868 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2869 emit_int16(0x44, (0xC0 | encode)); 2870 } 2871 2872 void Assembler::knotql(KRegister dst, KRegister src) { 2873 assert(VM_Version::supports_avx512bw(), ""); 2874 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2875 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2876 emit_int16(0x44, (0xC0 | encode)); 2877 } 2878 2879 // This instruction produces ZF or CF flags 2880 void Assembler::kortestbl(KRegister src1, KRegister src2) { 2881 assert(VM_Version::supports_avx512dq(), ""); 2882 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2883 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2884 emit_int16((unsigned char)0x98, (0xC0 | encode)); 2885 } 2886 2887 // This instruction produces ZF or CF flags 2888 void Assembler::kortestwl(KRegister src1, KRegister src2) { 2889 assert(VM_Version::supports_evex(), ""); 2890 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2891 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2892 emit_int16((unsigned char)0x98, (0xC0 | encode)); 2893 } 2894 2895 // This instruction produces ZF or CF flags 2896 void Assembler::kortestdl(KRegister src1, KRegister src2) { 2897 assert(VM_Version::supports_avx512bw(), ""); 2898 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2899 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2900 emit_int16((unsigned char)0x98, (0xC0 | encode)); 2901 } 2902 2903 // This instruction produces ZF or CF flags 2904 void Assembler::kortestql(KRegister src1, KRegister src2) { 2905 assert(VM_Version::supports_avx512bw(), ""); 2906 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2907 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2908 emit_int16((unsigned char)0x98, (0xC0 | encode)); 2909 } 2910 2911 // This instruction produces ZF or CF flags 2912 void Assembler::ktestql(KRegister src1, KRegister src2) { 2913 assert(VM_Version::supports_avx512bw(), ""); 2914 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2915 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2916 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2917 } 2918 2919 void Assembler::ktestdl(KRegister src1, KRegister src2) { 2920 assert(VM_Version::supports_avx512bw(), ""); 2921 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2922 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2923 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2924 } 2925 2926 void Assembler::ktestwl(KRegister src1, KRegister src2) { 2927 assert(VM_Version::supports_avx512dq(), ""); 2928 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2929 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2930 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2931 } 2932 2933 void Assembler::ktestbl(KRegister src1, KRegister src2) { 2934 assert(VM_Version::supports_avx512dq(), ""); 2935 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2936 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2937 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2938 } 2939 2940 void Assembler::ktestq(KRegister src1, KRegister src2) { 2941 assert(VM_Version::supports_avx512bw(), ""); 2942 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2943 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 2944 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2945 } 2946 2947 void Assembler::ktestd(KRegister src1, KRegister src2) { 2948 assert(VM_Version::supports_avx512bw(), ""); 2949 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2950 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2951 emit_int16((unsigned char)0x99, (0xC0 | encode)); 2952 } 2953 2954 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) { 2955 assert(VM_Version::supports_avx512dq(), ""); 2956 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2957 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 2958 emit_int16(0x46, (0xC0 | encode)); 2959 } 2960 2961 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) { 2962 assert(VM_Version::supports_avx512dq(), ""); 2963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2964 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2965 emit_int16(0x32, (0xC0 | encode)); 2966 emit_int8(imm8); 2967 } 2968 2969 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) { 2970 assert(VM_Version::supports_avx512bw(), ""); 2971 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2972 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2973 emit_int16(0x33, (0xC0 | encode)); 2974 emit_int8(imm8); 2975 } 2976 2977 2978 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) { 2979 assert(VM_Version::supports_avx512dq(), ""); 2980 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2981 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2982 emit_int16(0x30, (0xC0 | encode)); 2983 } 2984 2985 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) { 2986 assert(VM_Version::supports_evex(), ""); 2987 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2988 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2989 emit_int16(0x30, (0xC0 | encode)); 2990 emit_int8(imm8); 2991 } 2992 2993 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) { 2994 assert(VM_Version::supports_avx512bw(), ""); 2995 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 2996 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 2997 emit_int16(0x31, (0xC0 | encode)); 2998 emit_int8(imm8); 2999 } 3000 3001 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) { 3002 assert(VM_Version::supports_avx512bw(), ""); 3003 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3004 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 3005 emit_int16(0x31, (0xC0 | encode)); 3006 emit_int8(imm8); 3007 } 3008 3009 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) { 3010 assert(VM_Version::supports_avx512bw(), ""); 3011 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3012 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3013 emit_int16(0x4B, (0xC0 | encode)); 3014 } 3015 3016 void Assembler::movb(Address dst, int imm8) { 3017 InstructionMark im(this); 3018 prefix(dst); 3019 emit_int8((unsigned char)0xC6); 3020 emit_operand(rax, dst, 1); 3021 emit_int8(imm8); 3022 } 3023 3024 3025 void Assembler::movb(Address dst, Register src) { 3026 assert(src->has_byte_register(), "must have byte register"); 3027 InstructionMark im(this); 3028 prefix(dst, src, true); 3029 emit_int8((unsigned char)0x88); 3030 emit_operand(src, dst, 0); 3031 } 3032 3033 void Assembler::movdl(XMMRegister dst, Register src) { 3034 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3035 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3036 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3037 emit_int16(0x6E, (0xC0 | encode)); 3038 } 3039 3040 void Assembler::movdl(Register dst, XMMRegister src) { 3041 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3042 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3043 // swap src/dst to get correct prefix 3044 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3045 emit_int16(0x7E, (0xC0 | encode)); 3046 } 3047 3048 void Assembler::movdl(XMMRegister dst, Address src) { 3049 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3050 InstructionMark im(this); 3051 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3052 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3053 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3054 emit_int8(0x6E); 3055 emit_operand(dst, src, 0); 3056 } 3057 3058 void Assembler::movdl(Address dst, XMMRegister src) { 3059 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3060 InstructionMark im(this); 3061 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3062 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3063 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3064 emit_int8(0x7E); 3065 emit_operand(src, dst, 0); 3066 } 3067 3068 void Assembler::movdqa(XMMRegister dst, XMMRegister src) { 3069 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3070 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3071 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3072 emit_int16(0x6F, (0xC0 | encode)); 3073 } 3074 3075 void Assembler::movdqa(XMMRegister dst, Address src) { 3076 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3077 InstructionMark im(this); 3078 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3079 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3080 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3081 emit_int8(0x6F); 3082 emit_operand(dst, src, 0); 3083 } 3084 3085 void Assembler::movdqu(XMMRegister dst, Address src) { 3086 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3087 InstructionMark im(this); 3088 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3089 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3090 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3091 emit_int8(0x6F); 3092 emit_operand(dst, src, 0); 3093 } 3094 3095 void Assembler::movdqu(XMMRegister dst, XMMRegister src) { 3096 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3097 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3098 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3099 emit_int16(0x6F, (0xC0 | encode)); 3100 } 3101 3102 void Assembler::movdqu(Address dst, XMMRegister src) { 3103 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3104 InstructionMark im(this); 3105 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3106 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3107 attributes.reset_is_clear_context(); 3108 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3109 emit_int8(0x7F); 3110 emit_operand(src, dst, 0); 3111 } 3112 3113 // Move Unaligned 256bit Vector 3114 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3115 assert(UseAVX > 0, ""); 3116 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3117 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3118 emit_int16(0x6F, (0xC0 | encode)); 3119 } 3120 3121 void Assembler::vmovdqu(XMMRegister dst, Address src) { 3122 assert(UseAVX > 0, ""); 3123 InstructionMark im(this); 3124 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3125 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3126 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3127 emit_int8(0x6F); 3128 emit_operand(dst, src, 0); 3129 } 3130 3131 void Assembler::vmovdqu(Address dst, XMMRegister src) { 3132 assert(UseAVX > 0, ""); 3133 InstructionMark im(this); 3134 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3135 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3136 attributes.reset_is_clear_context(); 3137 // swap src<->dst for encoding 3138 assert(src != xnoreg, "sanity"); 3139 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3140 emit_int8(0x7F); 3141 emit_operand(src, dst, 0); 3142 } 3143 3144 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) { 3145 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), ""); 3146 InstructionMark im(this); 3147 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false); 3148 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3149 emit_int8((unsigned char)0x8C); 3150 emit_operand(dst, src, 0); 3151 } 3152 3153 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) { 3154 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), ""); 3155 InstructionMark im(this); 3156 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false); 3157 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3158 emit_int8((unsigned char)0x8C); 3159 emit_operand(dst, src, 0); 3160 } 3161 3162 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 3163 assert(UseAVX > 0, "requires some form of AVX"); 3164 InstructionMark im(this); 3165 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3166 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3167 emit_int8(0x2C); 3168 emit_operand(dst, src, 0); 3169 } 3170 3171 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 3172 assert(UseAVX > 0, "requires some form of AVX"); 3173 InstructionMark im(this); 3174 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3175 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3176 emit_int8(0x2D); 3177 emit_operand(dst, src, 0); 3178 } 3179 3180 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) { 3181 assert(UseAVX > 0, ""); 3182 InstructionMark im(this); 3183 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3184 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3185 emit_int8(0x2E); 3186 emit_operand(src, dst, 0); 3187 } 3188 3189 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) { 3190 assert(UseAVX > 0, ""); 3191 InstructionMark im(this); 3192 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 3193 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 3194 emit_int8(0x2F); 3195 emit_operand(src, dst, 0); 3196 } 3197 3198 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64) 3199 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3200 assert(VM_Version::supports_avx512vlbw(), ""); 3201 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3202 attributes.set_embedded_opmask_register_specifier(mask); 3203 attributes.set_is_evex_instruction(); 3204 if (merge) { 3205 attributes.reset_is_clear_context(); 3206 } 3207 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3208 emit_int16(0x6F, (0xC0 | encode)); 3209 } 3210 3211 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { 3212 // Unmasked instruction 3213 evmovdqub(dst, k0, src, /*merge*/ false, vector_len); 3214 } 3215 3216 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3217 assert(VM_Version::supports_avx512vlbw(), ""); 3218 InstructionMark im(this); 3219 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3220 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3221 attributes.set_embedded_opmask_register_specifier(mask); 3222 attributes.set_is_evex_instruction(); 3223 if (merge) { 3224 attributes.reset_is_clear_context(); 3225 } 3226 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3227 emit_int8(0x6F); 3228 emit_operand(dst, src, 0); 3229 } 3230 3231 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) { 3232 // Unmasked instruction 3233 evmovdqub(dst, k0, src, /*merge*/ false, vector_len); 3234 } 3235 3236 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3237 assert(VM_Version::supports_avx512vlbw(), ""); 3238 assert(src != xnoreg, "sanity"); 3239 InstructionMark im(this); 3240 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3241 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3242 attributes.set_embedded_opmask_register_specifier(mask); 3243 attributes.set_is_evex_instruction(); 3244 if (merge) { 3245 attributes.reset_is_clear_context(); 3246 } 3247 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3248 emit_int8(0x7F); 3249 emit_operand(src, dst, 0); 3250 } 3251 3252 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) { 3253 // Unmasked instruction 3254 evmovdquw(dst, k0, src, /*merge*/ false, vector_len); 3255 } 3256 3257 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3258 assert(VM_Version::supports_avx512vlbw(), ""); 3259 InstructionMark im(this); 3260 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3261 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3262 attributes.set_embedded_opmask_register_specifier(mask); 3263 attributes.set_is_evex_instruction(); 3264 if (merge) { 3265 attributes.reset_is_clear_context(); 3266 } 3267 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3268 emit_int8(0x6F); 3269 emit_operand(dst, src, 0); 3270 } 3271 3272 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) { 3273 // Unmasked instruction 3274 evmovdquw(dst, k0, src, /*merge*/ false, vector_len); 3275 } 3276 3277 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3278 assert(VM_Version::supports_avx512vlbw(), ""); 3279 assert(src != xnoreg, "sanity"); 3280 InstructionMark im(this); 3281 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 3282 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3283 attributes.set_embedded_opmask_register_specifier(mask); 3284 attributes.set_is_evex_instruction(); 3285 if (merge) { 3286 attributes.reset_is_clear_context(); 3287 } 3288 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3289 emit_int8(0x7F); 3290 emit_operand(src, dst, 0); 3291 } 3292 3293 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) { 3294 // Unmasked instruction 3295 evmovdqul(dst, k0, src, /*merge*/ false, vector_len); 3296 } 3297 3298 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3299 assert(VM_Version::supports_evex(), ""); 3300 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3301 attributes.set_embedded_opmask_register_specifier(mask); 3302 attributes.set_is_evex_instruction(); 3303 if (merge) { 3304 attributes.reset_is_clear_context(); 3305 } 3306 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3307 emit_int16(0x6F, (0xC0 | encode)); 3308 } 3309 3310 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) { 3311 // Unmasked instruction 3312 evmovdqul(dst, k0, src, /*merge*/ false, vector_len); 3313 } 3314 3315 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3316 assert(VM_Version::supports_evex(), ""); 3317 InstructionMark im(this); 3318 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true); 3319 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3320 attributes.set_embedded_opmask_register_specifier(mask); 3321 attributes.set_is_evex_instruction(); 3322 if (merge) { 3323 attributes.reset_is_clear_context(); 3324 } 3325 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3326 emit_int8(0x6F); 3327 emit_operand(dst, src, 0); 3328 } 3329 3330 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) { 3331 // Unmasked isntruction 3332 evmovdqul(dst, k0, src, /*merge*/ true, vector_len); 3333 } 3334 3335 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3336 assert(VM_Version::supports_evex(), ""); 3337 assert(src != xnoreg, "sanity"); 3338 InstructionMark im(this); 3339 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3340 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3341 attributes.set_embedded_opmask_register_specifier(mask); 3342 attributes.set_is_evex_instruction(); 3343 if (merge) { 3344 attributes.reset_is_clear_context(); 3345 } 3346 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3347 emit_int8(0x7F); 3348 emit_operand(src, dst, 0); 3349 } 3350 3351 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) { 3352 // Unmasked instruction 3353 evmovdquq(dst, k0, src, /*merge*/ false, vector_len); 3354 } 3355 3356 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3357 assert(VM_Version::supports_evex(), ""); 3358 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3359 attributes.set_embedded_opmask_register_specifier(mask); 3360 attributes.set_is_evex_instruction(); 3361 if (merge) { 3362 attributes.reset_is_clear_context(); 3363 } 3364 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3365 emit_int16(0x6F, (0xC0 | encode)); 3366 } 3367 3368 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) { 3369 // Unmasked instruction 3370 evmovdquq(dst, k0, src, /*merge*/ false, vector_len); 3371 } 3372 3373 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 3374 assert(VM_Version::supports_evex(), ""); 3375 InstructionMark im(this); 3376 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3377 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3378 attributes.set_embedded_opmask_register_specifier(mask); 3379 attributes.set_is_evex_instruction(); 3380 if (merge) { 3381 attributes.reset_is_clear_context(); 3382 } 3383 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3384 emit_int8(0x6F); 3385 emit_operand(dst, src, 0); 3386 } 3387 3388 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) { 3389 // Unmasked instruction 3390 evmovdquq(dst, k0, src, /*merge*/ true, vector_len); 3391 } 3392 3393 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 3394 assert(VM_Version::supports_evex(), ""); 3395 assert(src != xnoreg, "sanity"); 3396 InstructionMark im(this); 3397 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 3398 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 3399 attributes.set_embedded_opmask_register_specifier(mask); 3400 if (merge) { 3401 attributes.reset_is_clear_context(); 3402 } 3403 attributes.set_is_evex_instruction(); 3404 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3405 emit_int8(0x7F); 3406 emit_operand(src, dst, 0); 3407 } 3408 3409 // Uses zero extension on 64bit 3410 3411 void Assembler::movl(Register dst, int32_t imm32) { 3412 int encode = prefix_and_encode(dst->encoding()); 3413 emit_int8(0xB8 | encode); 3414 emit_int32(imm32); 3415 } 3416 3417 void Assembler::movl(Register dst, Register src) { 3418 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 3419 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 3420 } 3421 3422 void Assembler::movl(Register dst, Address src) { 3423 InstructionMark im(this); 3424 prefix(src, dst); 3425 emit_int8((unsigned char)0x8B); 3426 emit_operand(dst, src, 0); 3427 } 3428 3429 void Assembler::movl(Address dst, int32_t imm32) { 3430 InstructionMark im(this); 3431 prefix(dst); 3432 emit_int8((unsigned char)0xC7); 3433 emit_operand(rax, dst, 4); 3434 emit_int32(imm32); 3435 } 3436 3437 void Assembler::movl(Address dst, Register src) { 3438 InstructionMark im(this); 3439 prefix(dst, src); 3440 emit_int8((unsigned char)0x89); 3441 emit_operand(src, dst, 0); 3442 } 3443 3444 // New cpus require to use movsd and movss to avoid partial register stall 3445 // when loading from memory. But for old Opteron use movlpd instead of movsd. 3446 // The selection is done in MacroAssembler::movdbl() and movflt(). 3447 void Assembler::movlpd(XMMRegister dst, Address src) { 3448 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3449 InstructionMark im(this); 3450 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3451 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3452 attributes.set_rex_vex_w_reverted(); 3453 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3454 emit_int8(0x12); 3455 emit_operand(dst, src, 0); 3456 } 3457 3458 void Assembler::movq(XMMRegister dst, Address src) { 3459 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3460 InstructionMark im(this); 3461 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3462 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3463 attributes.set_rex_vex_w_reverted(); 3464 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3465 emit_int8(0x7E); 3466 emit_operand(dst, src, 0); 3467 } 3468 3469 void Assembler::movq(Address dst, XMMRegister src) { 3470 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3471 InstructionMark im(this); 3472 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3473 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3474 attributes.set_rex_vex_w_reverted(); 3475 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3476 emit_int8((unsigned char)0xD6); 3477 emit_operand(src, dst, 0); 3478 } 3479 3480 void Assembler::movq(XMMRegister dst, XMMRegister src) { 3481 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3482 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3483 attributes.set_rex_vex_w_reverted(); 3484 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3485 emit_int16((unsigned char)0xD6, (0xC0 | encode)); 3486 } 3487 3488 void Assembler::movq(Register dst, XMMRegister src) { 3489 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3490 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3491 // swap src/dst to get correct prefix 3492 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3493 emit_int16(0x7E, (0xC0 | encode)); 3494 } 3495 3496 void Assembler::movq(XMMRegister dst, Register src) { 3497 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3498 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3499 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 3500 emit_int16(0x6E, (0xC0 | encode)); 3501 } 3502 3503 void Assembler::movsbl(Register dst, Address src) { // movsxb 3504 InstructionMark im(this); 3505 prefix(src, dst); 3506 emit_int16(0x0F, (unsigned char)0xBE); 3507 emit_operand(dst, src, 0); 3508 } 3509 3510 void Assembler::movsbl(Register dst, Register src) { // movsxb 3511 NOT_LP64(assert(src->has_byte_register(), "must have byte register")); 3512 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true); 3513 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode)); 3514 } 3515 3516 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 3517 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3518 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3519 attributes.set_rex_vex_w_reverted(); 3520 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3521 emit_int16(0x10, (0xC0 | encode)); 3522 } 3523 3524 void Assembler::movsd(XMMRegister dst, Address src) { 3525 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3526 InstructionMark im(this); 3527 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3528 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3529 attributes.set_rex_vex_w_reverted(); 3530 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3531 emit_int8(0x10); 3532 emit_operand(dst, src, 0); 3533 } 3534 3535 void Assembler::movsd(Address dst, XMMRegister src) { 3536 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3537 InstructionMark im(this); 3538 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3539 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3540 attributes.reset_is_clear_context(); 3541 attributes.set_rex_vex_w_reverted(); 3542 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3543 emit_int8(0x11); 3544 emit_operand(src, dst, 0); 3545 } 3546 3547 void Assembler::movss(XMMRegister dst, XMMRegister src) { 3548 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3549 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3550 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3551 emit_int16(0x10, (0xC0 | encode)); 3552 } 3553 3554 void Assembler::movss(XMMRegister dst, Address src) { 3555 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3556 InstructionMark im(this); 3557 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3558 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3559 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3560 emit_int8(0x10); 3561 emit_operand(dst, src, 0); 3562 } 3563 3564 void Assembler::movss(Address dst, XMMRegister src) { 3565 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3566 InstructionMark im(this); 3567 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3568 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3569 attributes.reset_is_clear_context(); 3570 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3571 emit_int8(0x11); 3572 emit_operand(src, dst, 0); 3573 } 3574 3575 void Assembler::movswl(Register dst, Address src) { // movsxw 3576 InstructionMark im(this); 3577 prefix(src, dst); 3578 emit_int16(0x0F, (unsigned char)0xBF); 3579 emit_operand(dst, src, 0); 3580 } 3581 3582 void Assembler::movswl(Register dst, Register src) { // movsxw 3583 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 3584 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode)); 3585 } 3586 3587 void Assembler::movups(XMMRegister dst, Address src) { 3588 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3589 InstructionMark im(this); 3590 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3591 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit); 3592 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3593 emit_int8(0x10); 3594 emit_operand(dst, src, 0); 3595 } 3596 3597 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) { 3598 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 3599 InstructionMark im(this); 3600 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3601 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit); 3602 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3603 emit_int8(0x10); 3604 emit_operand(dst, src, 0); 3605 } 3606 3607 void Assembler::movups(Address dst, XMMRegister src) { 3608 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3609 InstructionMark im(this); 3610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3611 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit); 3612 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3613 emit_int8(0x11); 3614 emit_operand(src, dst, 0); 3615 } 3616 3617 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) { 3618 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 3619 InstructionMark im(this); 3620 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 3621 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit); 3622 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 3623 emit_int8(0x11); 3624 emit_operand(src, dst, 0); 3625 } 3626 3627 void Assembler::movw(Address dst, int imm16) { 3628 InstructionMark im(this); 3629 3630 emit_int8(0x66); // switch to 16-bit mode 3631 prefix(dst); 3632 emit_int8((unsigned char)0xC7); 3633 emit_operand(rax, dst, 2); 3634 emit_int16(imm16); 3635 } 3636 3637 void Assembler::movw(Register dst, Address src) { 3638 InstructionMark im(this); 3639 emit_int8(0x66); 3640 prefix(src, dst); 3641 emit_int8((unsigned char)0x8B); 3642 emit_operand(dst, src, 0); 3643 } 3644 3645 void Assembler::movw(Address dst, Register src) { 3646 InstructionMark im(this); 3647 emit_int8(0x66); 3648 prefix(dst, src); 3649 emit_int8((unsigned char)0x89); 3650 emit_operand(src, dst, 0); 3651 } 3652 3653 void Assembler::movzbl(Register dst, Address src) { // movzxb 3654 InstructionMark im(this); 3655 prefix(src, dst); 3656 emit_int16(0x0F, (unsigned char)0xB6); 3657 emit_operand(dst, src, 0); 3658 } 3659 3660 void Assembler::movzbl(Register dst, Register src) { // movzxb 3661 NOT_LP64(assert(src->has_byte_register(), "must have byte register")); 3662 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true); 3663 emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode); 3664 } 3665 3666 void Assembler::movzwl(Register dst, Address src) { // movzxw 3667 InstructionMark im(this); 3668 prefix(src, dst); 3669 emit_int16(0x0F, (unsigned char)0xB7); 3670 emit_operand(dst, src, 0); 3671 } 3672 3673 void Assembler::movzwl(Register dst, Register src) { // movzxw 3674 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 3675 emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode); 3676 } 3677 3678 void Assembler::mull(Address src) { 3679 InstructionMark im(this); 3680 prefix(src); 3681 emit_int8((unsigned char)0xF7); 3682 emit_operand(rsp, src, 0); 3683 } 3684 3685 void Assembler::mull(Register src) { 3686 int encode = prefix_and_encode(src->encoding()); 3687 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 3688 } 3689 3690 void Assembler::mulsd(XMMRegister dst, Address src) { 3691 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3692 InstructionMark im(this); 3693 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3694 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 3695 attributes.set_rex_vex_w_reverted(); 3696 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3697 emit_int8(0x59); 3698 emit_operand(dst, src, 0); 3699 } 3700 3701 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 3702 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 3703 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3704 attributes.set_rex_vex_w_reverted(); 3705 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 3706 emit_int16(0x59, (0xC0 | encode)); 3707 } 3708 3709 void Assembler::mulss(XMMRegister dst, Address src) { 3710 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3711 InstructionMark im(this); 3712 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3713 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 3714 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3715 emit_int8(0x59); 3716 emit_operand(dst, src, 0); 3717 } 3718 3719 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 3720 NOT_LP64(assert(VM_Version::supports_sse(), "")); 3721 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 3722 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 3723 emit_int16(0x59, (0xC0 | encode)); 3724 } 3725 3726 void Assembler::negl(Register dst) { 3727 int encode = prefix_and_encode(dst->encoding()); 3728 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 3729 } 3730 3731 void Assembler::negl(Address dst) { 3732 InstructionMark im(this); 3733 prefix(dst); 3734 emit_int8((unsigned char)0xF7); 3735 emit_operand(as_Register(3), dst, 0); 3736 } 3737 3738 void Assembler::nop(int i) { 3739 #ifdef ASSERT 3740 assert(i > 0, " "); 3741 // The fancy nops aren't currently recognized by debuggers making it a 3742 // pain to disassemble code while debugging. If asserts are on clearly 3743 // speed is not an issue so simply use the single byte traditional nop 3744 // to do alignment. 3745 3746 for (; i > 0 ; i--) emit_int8((unsigned char)0x90); 3747 return; 3748 3749 #endif // ASSERT 3750 3751 if (UseAddressNop && VM_Version::is_intel()) { 3752 // 3753 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel 3754 // 1: 0x90 3755 // 2: 0x66 0x90 3756 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 3757 // 4: 0x0F 0x1F 0x40 0x00 3758 // 5: 0x0F 0x1F 0x44 0x00 0x00 3759 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 3760 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 3761 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3762 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3763 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3764 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3765 3766 // The rest coding is Intel specific - don't use consecutive address nops 3767 3768 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3769 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3770 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3771 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3772 3773 while(i >= 15) { 3774 // For Intel don't generate consecutive address nops (mix with regular nops) 3775 i -= 15; 3776 emit_int24(0x66, 0x66, 0x66); 3777 addr_nop_8(); 3778 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 3779 } 3780 switch (i) { 3781 case 14: 3782 emit_int8(0x66); // size prefix 3783 case 13: 3784 emit_int8(0x66); // size prefix 3785 case 12: 3786 addr_nop_8(); 3787 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 3788 break; 3789 case 11: 3790 emit_int8(0x66); // size prefix 3791 case 10: 3792 emit_int8(0x66); // size prefix 3793 case 9: 3794 emit_int8(0x66); // size prefix 3795 case 8: 3796 addr_nop_8(); 3797 break; 3798 case 7: 3799 addr_nop_7(); 3800 break; 3801 case 6: 3802 emit_int8(0x66); // size prefix 3803 case 5: 3804 addr_nop_5(); 3805 break; 3806 case 4: 3807 addr_nop_4(); 3808 break; 3809 case 3: 3810 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 3811 emit_int8(0x66); // size prefix 3812 case 2: 3813 emit_int8(0x66); // size prefix 3814 case 1: 3815 emit_int8((unsigned char)0x90); 3816 // nop 3817 break; 3818 default: 3819 assert(i == 0, " "); 3820 } 3821 return; 3822 } 3823 if (UseAddressNop && VM_Version::is_amd_family()) { 3824 // 3825 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD. 3826 // 1: 0x90 3827 // 2: 0x66 0x90 3828 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 3829 // 4: 0x0F 0x1F 0x40 0x00 3830 // 5: 0x0F 0x1F 0x44 0x00 0x00 3831 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 3832 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 3833 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3834 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3835 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3836 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3837 3838 // The rest coding is AMD specific - use consecutive address nops 3839 3840 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00 3841 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00 3842 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 3843 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 3844 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3845 // Size prefixes (0x66) are added for larger sizes 3846 3847 while(i >= 22) { 3848 i -= 11; 3849 emit_int24(0x66, 0x66, 0x66); 3850 addr_nop_8(); 3851 } 3852 // Generate first nop for size between 21-12 3853 switch (i) { 3854 case 21: 3855 i -= 1; 3856 emit_int8(0x66); // size prefix 3857 case 20: 3858 case 19: 3859 i -= 1; 3860 emit_int8(0x66); // size prefix 3861 case 18: 3862 case 17: 3863 i -= 1; 3864 emit_int8(0x66); // size prefix 3865 case 16: 3866 case 15: 3867 i -= 8; 3868 addr_nop_8(); 3869 break; 3870 case 14: 3871 case 13: 3872 i -= 7; 3873 addr_nop_7(); 3874 break; 3875 case 12: 3876 i -= 6; 3877 emit_int8(0x66); // size prefix 3878 addr_nop_5(); 3879 break; 3880 default: 3881 assert(i < 12, " "); 3882 } 3883 3884 // Generate second nop for size between 11-1 3885 switch (i) { 3886 case 11: 3887 emit_int8(0x66); // size prefix 3888 case 10: 3889 emit_int8(0x66); // size prefix 3890 case 9: 3891 emit_int8(0x66); // size prefix 3892 case 8: 3893 addr_nop_8(); 3894 break; 3895 case 7: 3896 addr_nop_7(); 3897 break; 3898 case 6: 3899 emit_int8(0x66); // size prefix 3900 case 5: 3901 addr_nop_5(); 3902 break; 3903 case 4: 3904 addr_nop_4(); 3905 break; 3906 case 3: 3907 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 3908 emit_int8(0x66); // size prefix 3909 case 2: 3910 emit_int8(0x66); // size prefix 3911 case 1: 3912 emit_int8((unsigned char)0x90); 3913 // nop 3914 break; 3915 default: 3916 assert(i == 0, " "); 3917 } 3918 return; 3919 } 3920 3921 if (UseAddressNop && VM_Version::is_zx()) { 3922 // 3923 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX 3924 // 1: 0x90 3925 // 2: 0x66 0x90 3926 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding) 3927 // 4: 0x0F 0x1F 0x40 0x00 3928 // 5: 0x0F 0x1F 0x44 0x00 0x00 3929 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00 3930 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 3931 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3932 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3933 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3934 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 3935 3936 // The rest coding is ZX specific - don't use consecutive address nops 3937 3938 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3939 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3940 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3941 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90 3942 3943 while (i >= 15) { 3944 // For ZX don't generate consecutive address nops (mix with regular nops) 3945 i -= 15; 3946 emit_int24(0x66, 0x66, 0x66); 3947 addr_nop_8(); 3948 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 3949 } 3950 switch (i) { 3951 case 14: 3952 emit_int8(0x66); // size prefix 3953 case 13: 3954 emit_int8(0x66); // size prefix 3955 case 12: 3956 addr_nop_8(); 3957 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 3958 break; 3959 case 11: 3960 emit_int8(0x66); // size prefix 3961 case 10: 3962 emit_int8(0x66); // size prefix 3963 case 9: 3964 emit_int8(0x66); // size prefix 3965 case 8: 3966 addr_nop_8(); 3967 break; 3968 case 7: 3969 addr_nop_7(); 3970 break; 3971 case 6: 3972 emit_int8(0x66); // size prefix 3973 case 5: 3974 addr_nop_5(); 3975 break; 3976 case 4: 3977 addr_nop_4(); 3978 break; 3979 case 3: 3980 // Don't use "0x0F 0x1F 0x00" - need patching safe padding 3981 emit_int8(0x66); // size prefix 3982 case 2: 3983 emit_int8(0x66); // size prefix 3984 case 1: 3985 emit_int8((unsigned char)0x90); 3986 // nop 3987 break; 3988 default: 3989 assert(i == 0, " "); 3990 } 3991 return; 3992 } 3993 3994 // Using nops with size prefixes "0x66 0x90". 3995 // From AMD Optimization Guide: 3996 // 1: 0x90 3997 // 2: 0x66 0x90 3998 // 3: 0x66 0x66 0x90 3999 // 4: 0x66 0x66 0x66 0x90 4000 // 5: 0x66 0x66 0x90 0x66 0x90 4001 // 6: 0x66 0x66 0x90 0x66 0x66 0x90 4002 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 4003 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90 4004 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90 4005 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90 4006 // 4007 while (i > 12) { 4008 i -= 4; 4009 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90); 4010 } 4011 // 1 - 12 nops 4012 if (i > 8) { 4013 if (i > 9) { 4014 i -= 1; 4015 emit_int8(0x66); 4016 } 4017 i -= 3; 4018 emit_int24(0x66, 0x66, (unsigned char)0x90); 4019 } 4020 // 1 - 8 nops 4021 if (i > 4) { 4022 if (i > 6) { 4023 i -= 1; 4024 emit_int8(0x66); 4025 } 4026 i -= 3; 4027 emit_int24(0x66, 0x66, (unsigned char)0x90); 4028 } 4029 switch (i) { 4030 case 4: 4031 emit_int8(0x66); 4032 case 3: 4033 emit_int8(0x66); 4034 case 2: 4035 emit_int8(0x66); 4036 case 1: 4037 emit_int8((unsigned char)0x90); 4038 break; 4039 default: 4040 assert(i == 0, " "); 4041 } 4042 } 4043 4044 void Assembler::notl(Register dst) { 4045 int encode = prefix_and_encode(dst->encoding()); 4046 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 4047 } 4048 4049 void Assembler::orw(Register dst, Register src) { 4050 (void)prefix_and_encode(dst->encoding(), src->encoding()); 4051 emit_arith(0x0B, 0xC0, dst, src); 4052 } 4053 4054 void Assembler::orl(Address dst, int32_t imm32) { 4055 InstructionMark im(this); 4056 prefix(dst); 4057 emit_arith_operand(0x81, rcx, dst, imm32); 4058 } 4059 4060 void Assembler::orl(Register dst, int32_t imm32) { 4061 prefix(dst); 4062 emit_arith(0x81, 0xC8, dst, imm32); 4063 } 4064 4065 void Assembler::orl(Register dst, Address src) { 4066 InstructionMark im(this); 4067 prefix(src, dst); 4068 emit_int8(0x0B); 4069 emit_operand(dst, src, 0); 4070 } 4071 4072 void Assembler::orl(Register dst, Register src) { 4073 (void) prefix_and_encode(dst->encoding(), src->encoding()); 4074 emit_arith(0x0B, 0xC0, dst, src); 4075 } 4076 4077 void Assembler::orl(Address dst, Register src) { 4078 InstructionMark im(this); 4079 prefix(dst, src); 4080 emit_int8(0x09); 4081 emit_operand(src, dst, 0); 4082 } 4083 4084 void Assembler::orb(Address dst, int imm8) { 4085 InstructionMark im(this); 4086 prefix(dst); 4087 emit_int8((unsigned char)0x80); 4088 emit_operand(rcx, dst, 1); 4089 emit_int8(imm8); 4090 } 4091 4092 void Assembler::orb(Address dst, Register src) { 4093 InstructionMark im(this); 4094 prefix(dst, src, true); 4095 emit_int8(0x08); 4096 emit_operand(src, dst, 0); 4097 } 4098 4099 void Assembler::packsswb(XMMRegister dst, XMMRegister src) { 4100 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 4101 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4102 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4103 emit_int16(0x63, (0xC0 | encode)); 4104 } 4105 4106 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4107 assert(UseAVX > 0, "some form of AVX must be enabled"); 4108 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4109 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4110 emit_int16(0x63, (0xC0 | encode)); 4111 } 4112 4113 void Assembler::packssdw(XMMRegister dst, XMMRegister src) { 4114 assert(VM_Version::supports_sse2(), ""); 4115 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4116 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4117 emit_int16(0x6B, (0xC0 | encode)); 4118 } 4119 4120 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4121 assert(UseAVX > 0, "some form of AVX must be enabled"); 4122 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4123 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4124 emit_int16(0x6B, (0xC0 | encode)); 4125 } 4126 4127 void Assembler::packuswb(XMMRegister dst, Address src) { 4128 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 4129 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 4130 InstructionMark im(this); 4131 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4132 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 4133 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4134 emit_int8(0x67); 4135 emit_operand(dst, src, 0); 4136 } 4137 4138 void Assembler::packuswb(XMMRegister dst, XMMRegister src) { 4139 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 4140 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4141 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4142 emit_int16(0x67, (0xC0 | encode)); 4143 } 4144 4145 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4146 assert(UseAVX > 0, "some form of AVX must be enabled"); 4147 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4148 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4149 emit_int16(0x67, (0xC0 | encode)); 4150 } 4151 4152 void Assembler::packusdw(XMMRegister dst, XMMRegister src) { 4153 assert(VM_Version::supports_sse4_1(), ""); 4154 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4155 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4156 emit_int16(0x2B, (0xC0 | encode)); 4157 } 4158 4159 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4160 assert(UseAVX > 0, "some form of AVX must be enabled"); 4161 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4162 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4163 emit_int16(0x2B, (0xC0 | encode)); 4164 } 4165 4166 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4167 assert(VM_Version::supports_avx2(), ""); 4168 assert(vector_len != AVX_128bit, ""); 4169 // VEX.256.66.0F3A.W1 00 /r ib 4170 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4171 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4172 emit_int24(0x00, (0xC0 | encode), imm8); 4173 } 4174 4175 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4176 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() : 4177 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported"); 4178 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4179 attributes.set_is_evex_instruction(); 4180 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4181 emit_int16(0x36, (0xC0 | encode)); 4182 } 4183 4184 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4185 assert(VM_Version::supports_avx512_vbmi(), ""); 4186 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4187 attributes.set_is_evex_instruction(); 4188 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4189 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 4190 } 4191 4192 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 4193 assert(VM_Version::supports_avx512_vbmi(), ""); 4194 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4195 attributes.set_is_evex_instruction(); 4196 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4197 emit_int8((unsigned char)0x8D); 4198 emit_operand(dst, src, 0); 4199 } 4200 4201 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4202 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() : 4203 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() : 4204 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported"); 4205 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4206 attributes.set_is_evex_instruction(); 4207 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4208 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 4209 } 4210 4211 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4212 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), ""); 4213 // VEX.NDS.256.66.0F38.W0 36 /r 4214 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4215 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4216 emit_int16(0x36, (0xC0 | encode)); 4217 } 4218 4219 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 4220 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), ""); 4221 // VEX.NDS.256.66.0F38.W0 36 /r 4222 InstructionMark im(this); 4223 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4224 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4225 emit_int8(0x36); 4226 emit_operand(dst, src, 0); 4227 } 4228 4229 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 4230 assert(VM_Version::supports_avx2(), ""); 4231 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4232 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4233 emit_int24(0x46, (0xC0 | encode), imm8); 4234 } 4235 4236 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 4237 assert(VM_Version::supports_avx(), ""); 4238 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4240 emit_int24(0x06, (0xC0 | encode), imm8); 4241 } 4242 4243 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4244 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 4245 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4246 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4247 emit_int24(0x04, (0xC0 | encode), imm8); 4248 } 4249 4250 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4251 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), ""); 4252 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false); 4253 attributes.set_rex_vex_w_reverted(); 4254 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4255 emit_int24(0x05, (0xC0 | encode), imm8); 4256 } 4257 4258 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) { 4259 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), ""); 4260 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false); 4261 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4262 emit_int24(0x01, (0xC0 | encode), imm8); 4263 } 4264 4265 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4266 assert(VM_Version::supports_evex(), ""); 4267 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4268 attributes.set_is_evex_instruction(); 4269 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4270 emit_int16(0x76, (0xC0 | encode)); 4271 } 4272 4273 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4274 assert(VM_Version::supports_avx512_vbmi(), ""); 4275 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4276 attributes.set_is_evex_instruction(); 4277 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4278 emit_int16(0x7D, (0xC0 | encode)); 4279 } 4280 4281 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) { 4282 assert(VM_Version::supports_avx512_vbmi(), ""); 4283 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4284 attributes.set_is_evex_instruction(); 4285 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4286 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode)); 4287 } 4288 4289 void Assembler::pause() { 4290 emit_int16((unsigned char)0xF3, (unsigned char)0x90); 4291 } 4292 4293 void Assembler::ud2() { 4294 emit_int16(0x0F, 0x0B); 4295 } 4296 4297 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 4298 assert(VM_Version::supports_sse4_2(), ""); 4299 InstructionMark im(this); 4300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4301 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4302 emit_int8(0x61); 4303 emit_operand(dst, src, 1); 4304 emit_int8(imm8); 4305 } 4306 4307 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 4308 assert(VM_Version::supports_sse4_2(), ""); 4309 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4310 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4311 emit_int24(0x61, (0xC0 | encode), imm8); 4312 } 4313 4314 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4315 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 4316 assert(VM_Version::supports_sse2(), ""); 4317 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4318 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4319 emit_int16(0x74, (0xC0 | encode)); 4320 } 4321 4322 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) { 4323 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4324 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4325 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4326 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4327 emit_int16(cond_encoding, (0xC0 | encode)); 4328 } 4329 4330 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4331 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4332 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4333 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4334 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4335 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4336 emit_int16(0x74, (0xC0 | encode)); 4337 } 4338 4339 // In this context, kdst is written the mask used to process the equal components 4340 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 4341 assert(VM_Version::supports_avx512bw(), ""); 4342 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4343 attributes.set_is_evex_instruction(); 4344 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4345 emit_int16(0x74, (0xC0 | encode)); 4346 } 4347 4348 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4349 assert(VM_Version::supports_avx512vlbw(), ""); 4350 InstructionMark im(this); 4351 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4352 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4353 attributes.set_is_evex_instruction(); 4354 int dst_enc = kdst->encoding(); 4355 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4356 emit_int8(0x64); 4357 emit_operand(as_Register(dst_enc), src, 0); 4358 } 4359 4360 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 4361 assert(VM_Version::supports_avx512vlbw(), ""); 4362 InstructionMark im(this); 4363 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4364 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4365 attributes.reset_is_clear_context(); 4366 attributes.set_embedded_opmask_register_specifier(mask); 4367 attributes.set_is_evex_instruction(); 4368 int dst_enc = kdst->encoding(); 4369 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4370 emit_int8(0x64); 4371 emit_operand(as_Register(dst_enc), src, 0); 4372 } 4373 4374 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { 4375 assert(VM_Version::supports_avx512vlbw(), ""); 4376 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4377 attributes.set_is_evex_instruction(); 4378 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4379 emit_int24(0x3E, (0xC0 | encode), vcc); 4380 } 4381 4382 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) { 4383 assert(VM_Version::supports_avx512vlbw(), ""); 4384 InstructionMark im(this); 4385 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4386 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4387 attributes.set_is_evex_instruction(); 4388 int dst_enc = kdst->encoding(); 4389 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4390 emit_int8(0x3E); 4391 emit_operand(as_Register(dst_enc), src, 1); 4392 emit_int8(vcc); 4393 } 4394 4395 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4396 assert(VM_Version::supports_avx512bw(), ""); 4397 InstructionMark im(this); 4398 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4399 attributes.set_is_evex_instruction(); 4400 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4401 int dst_enc = kdst->encoding(); 4402 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4403 emit_int8(0x74); 4404 emit_operand(as_Register(dst_enc), src, 0); 4405 } 4406 4407 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 4408 assert(VM_Version::supports_avx512vlbw(), ""); 4409 InstructionMark im(this); 4410 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4411 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4412 attributes.reset_is_clear_context(); 4413 attributes.set_embedded_opmask_register_specifier(mask); 4414 attributes.set_is_evex_instruction(); 4415 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4416 emit_int8(0x74); 4417 emit_operand(as_Register(kdst->encoding()), src, 0); 4418 } 4419 4420 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4421 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 4422 assert(VM_Version::supports_sse2(), ""); 4423 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4424 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4425 emit_int16(0x75, (0xC0 | encode)); 4426 } 4427 4428 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4429 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4430 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4431 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4432 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4433 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4434 emit_int16(0x75, (0xC0 | encode)); 4435 } 4436 4437 // In this context, kdst is written the mask used to process the equal components 4438 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 4439 assert(VM_Version::supports_avx512bw(), ""); 4440 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4441 attributes.set_is_evex_instruction(); 4442 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4443 emit_int16(0x75, (0xC0 | encode)); 4444 } 4445 4446 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4447 assert(VM_Version::supports_avx512bw(), ""); 4448 InstructionMark im(this); 4449 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4450 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 4451 attributes.set_is_evex_instruction(); 4452 int dst_enc = kdst->encoding(); 4453 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4454 emit_int8(0x75); 4455 emit_operand(as_Register(dst_enc), src, 0); 4456 } 4457 4458 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4459 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) { 4460 assert(VM_Version::supports_sse2(), ""); 4461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4462 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4463 emit_int16(0x76, (0xC0 | encode)); 4464 } 4465 4466 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4467 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4468 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 4469 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 4470 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4471 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4472 emit_int16(0x76, (0xC0 | encode)); 4473 } 4474 4475 // In this context, kdst is written the mask used to process the equal components 4476 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) { 4477 assert(VM_Version::supports_evex(), ""); 4478 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4479 attributes.set_is_evex_instruction(); 4480 attributes.reset_is_clear_context(); 4481 attributes.set_embedded_opmask_register_specifier(mask); 4482 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4483 emit_int16(0x76, (0xC0 | encode)); 4484 } 4485 4486 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) { 4487 assert(VM_Version::supports_evex(), ""); 4488 InstructionMark im(this); 4489 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4490 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 4491 attributes.set_is_evex_instruction(); 4492 attributes.reset_is_clear_context(); 4493 attributes.set_embedded_opmask_register_specifier(mask); 4494 int dst_enc = kdst->encoding(); 4495 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4496 emit_int8(0x76); 4497 emit_operand(as_Register(dst_enc), src, 0); 4498 } 4499 4500 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4501 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) { 4502 assert(VM_Version::supports_sse4_1(), ""); 4503 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4504 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4505 emit_int16(0x29, (0xC0 | encode)); 4506 } 4507 4508 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) { 4509 assert(VM_Version::supports_evex(), ""); 4510 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4511 attributes.set_is_evex_instruction(); 4512 attributes.reset_is_clear_context(); 4513 attributes.set_embedded_opmask_register_specifier(mask); 4514 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4515 emit_int16(0x29, (0xC0 | encode)); 4516 } 4517 4518 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) { 4519 assert(VM_Version::supports_avx(), ""); 4520 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4521 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4522 emit_int16(cond_encoding, (0xC0 | encode)); 4523 } 4524 4525 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst 4526 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4527 assert(VM_Version::supports_avx(), ""); 4528 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4530 emit_int16(0x29, (0xC0 | encode)); 4531 } 4532 4533 // In this context, kdst is written the mask used to process the equal components 4534 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) { 4535 assert(VM_Version::supports_evex(), ""); 4536 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4537 attributes.reset_is_clear_context(); 4538 attributes.set_is_evex_instruction(); 4539 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4540 emit_int16(0x29, (0xC0 | encode)); 4541 } 4542 4543 // In this context, kdst is written the mask used to process the equal components 4544 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) { 4545 assert(VM_Version::supports_evex(), ""); 4546 InstructionMark im(this); 4547 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4548 attributes.reset_is_clear_context(); 4549 attributes.set_is_evex_instruction(); 4550 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 4551 int dst_enc = kdst->encoding(); 4552 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4553 emit_int8(0x29); 4554 emit_operand(as_Register(dst_enc), src, 0); 4555 } 4556 4557 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) { 4558 assert(VM_Version::supports_sse4_1(), ""); 4559 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4560 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4561 emit_int16(0x37, (0xC0 | encode)); 4562 } 4563 4564 void Assembler::pmovmskb(Register dst, XMMRegister src) { 4565 assert(VM_Version::supports_sse2(), ""); 4566 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4567 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4568 emit_int16((unsigned char)0xD7, (0xC0 | encode)); 4569 } 4570 4571 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) { 4572 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) || 4573 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), ""); 4574 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4575 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4576 emit_int16((unsigned char)0xD7, (0xC0 | encode)); 4577 } 4578 4579 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) { 4580 assert(VM_Version::supports_avx(), ""); 4581 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4582 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 4583 emit_int16(0x50, (0xC0 | encode)); 4584 } 4585 4586 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) { 4587 assert(VM_Version::supports_avx(), ""); 4588 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 4589 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4590 emit_int16(0x50, (0xC0 | encode)); 4591 } 4592 4593 4594 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) { 4595 assert(VM_Version::supports_sse4_1(), ""); 4596 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4597 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4598 emit_int24(0x16, (0xC0 | encode), imm8); 4599 } 4600 4601 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) { 4602 assert(VM_Version::supports_sse4_1(), ""); 4603 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4604 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 4605 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4606 emit_int8(0x16); 4607 emit_operand(src, dst, 1); 4608 emit_int8(imm8); 4609 } 4610 4611 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) { 4612 assert(VM_Version::supports_sse4_1(), ""); 4613 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4614 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4615 emit_int24(0x16, (0xC0 | encode), imm8); 4616 } 4617 4618 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) { 4619 assert(VM_Version::supports_sse4_1(), ""); 4620 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4621 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 4622 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4623 emit_int8(0x16); 4624 emit_operand(src, dst, 1); 4625 emit_int8(imm8); 4626 } 4627 4628 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) { 4629 assert(VM_Version::supports_sse2(), ""); 4630 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4631 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4632 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8); 4633 } 4634 4635 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) { 4636 assert(VM_Version::supports_sse4_1(), ""); 4637 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4638 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 4639 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4640 emit_int8(0x15); 4641 emit_operand(src, dst, 1); 4642 emit_int8(imm8); 4643 } 4644 4645 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) { 4646 assert(VM_Version::supports_sse4_1(), ""); 4647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4648 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4649 emit_int24(0x14, (0xC0 | encode), imm8); 4650 } 4651 4652 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) { 4653 assert(VM_Version::supports_sse4_1(), ""); 4654 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4655 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 4656 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4657 emit_int8(0x14); 4658 emit_operand(src, dst, 1); 4659 emit_int8(imm8); 4660 } 4661 4662 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) { 4663 assert(VM_Version::supports_sse4_1(), ""); 4664 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4665 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4666 emit_int24(0x22, (0xC0 | encode), imm8); 4667 } 4668 4669 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) { 4670 assert(VM_Version::supports_sse4_1(), ""); 4671 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4672 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 4673 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4674 emit_int8(0x22); 4675 emit_operand(dst, src, 1); 4676 emit_int8(imm8); 4677 } 4678 4679 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 4680 assert(VM_Version::supports_avx(), ""); 4681 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4683 emit_int24(0x22, (0xC0 | encode), imm8); 4684 } 4685 4686 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) { 4687 assert(VM_Version::supports_sse4_1(), ""); 4688 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4689 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4690 emit_int24(0x22, (0xC0 | encode), imm8); 4691 } 4692 4693 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) { 4694 assert(VM_Version::supports_sse4_1(), ""); 4695 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4696 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 4697 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4698 emit_int8(0x22); 4699 emit_operand(dst, src, 1); 4700 emit_int8(imm8); 4701 } 4702 4703 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 4704 assert(VM_Version::supports_avx(), ""); 4705 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false); 4706 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4707 emit_int24(0x22, (0xC0 | encode), imm8); 4708 } 4709 4710 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) { 4711 assert(VM_Version::supports_sse2(), ""); 4712 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4713 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4714 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8); 4715 } 4716 4717 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) { 4718 assert(VM_Version::supports_sse2(), ""); 4719 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4720 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 4721 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4722 emit_int8((unsigned char)0xC4); 4723 emit_operand(dst, src, 1); 4724 emit_int8(imm8); 4725 } 4726 4727 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 4728 assert(VM_Version::supports_avx(), ""); 4729 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4730 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4731 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8); 4732 } 4733 4734 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) { 4735 assert(VM_Version::supports_sse4_1(), ""); 4736 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4737 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 4738 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4739 emit_int8(0x20); 4740 emit_operand(dst, src, 1); 4741 emit_int8(imm8); 4742 } 4743 4744 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) { 4745 assert(VM_Version::supports_sse4_1(), ""); 4746 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4747 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4748 emit_int24(0x20, (0xC0 | encode), imm8); 4749 } 4750 4751 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) { 4752 assert(VM_Version::supports_avx(), ""); 4753 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false); 4754 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4755 emit_int24(0x20, (0xC0 | encode), imm8); 4756 } 4757 4758 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) { 4759 assert(VM_Version::supports_sse4_1(), ""); 4760 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4761 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4762 emit_int24(0x21, (0xC0 | encode), imm8); 4763 } 4764 4765 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) { 4766 assert(VM_Version::supports_avx(), ""); 4767 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 4768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 4769 emit_int24(0x21, (0xC0 | encode), imm8); 4770 } 4771 4772 void Assembler::pmovzxbw(XMMRegister dst, Address src) { 4773 assert(VM_Version::supports_sse4_1(), ""); 4774 InstructionMark im(this); 4775 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4776 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4777 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4778 emit_int8(0x30); 4779 emit_operand(dst, src, 0); 4780 } 4781 4782 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 4783 assert(VM_Version::supports_sse4_1(), ""); 4784 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4785 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4786 emit_int16(0x30, (0xC0 | encode)); 4787 } 4788 4789 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) { 4790 assert(VM_Version::supports_sse4_1(), ""); 4791 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4792 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4793 emit_int16(0x20, (0xC0 | encode)); 4794 } 4795 4796 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) { 4797 assert(VM_Version::supports_sse4_1(), ""); 4798 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4799 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4800 emit_int16(0x35, (0xC0 | encode)); 4801 } 4802 4803 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) { 4804 assert(VM_Version::supports_sse4_1(), ""); 4805 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4806 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4807 emit_int16(0x21, (0xC0 | encode)); 4808 } 4809 4810 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) { 4811 assert(VM_Version::supports_sse4_1(), ""); 4812 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4813 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4814 emit_int16(0x31, (0xC0 | encode)); 4815 } 4816 4817 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) { 4818 assert(VM_Version::supports_sse4_1(), ""); 4819 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4820 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4821 emit_int16(0x22, (0xC0 | encode)); 4822 } 4823 4824 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) { 4825 assert(VM_Version::supports_sse4_1(), ""); 4826 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4827 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4828 emit_int16(0x23, (0xC0 | encode)); 4829 } 4830 4831 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 4832 assert(VM_Version::supports_avx(), ""); 4833 InstructionMark im(this); 4834 assert(dst != xnoreg, "sanity"); 4835 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4836 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4837 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4838 emit_int8(0x30); 4839 emit_operand(dst, src, 0); 4840 } 4841 4842 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { 4843 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 4844 vector_len == AVX_256bit? VM_Version::supports_avx2() : 4845 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 4846 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4847 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4848 emit_int16(0x30, (unsigned char) (0xC0 | encode)); 4849 } 4850 4851 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) { 4852 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 4853 vector_len == AVX_256bit? VM_Version::supports_avx2() : 4854 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 4855 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 4856 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4857 emit_int16(0x20, (0xC0 | encode)); 4858 } 4859 4860 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) { 4861 assert(VM_Version::supports_avx512vlbw(), ""); 4862 assert(dst != xnoreg, "sanity"); 4863 InstructionMark im(this); 4864 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 4865 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4866 attributes.set_embedded_opmask_register_specifier(mask); 4867 attributes.set_is_evex_instruction(); 4868 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4869 emit_int8(0x30); 4870 emit_operand(dst, src, 0); 4871 } 4872 4873 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 4874 assert(VM_Version::supports_avx512vl(), ""); 4875 assert(dst != xnoreg, "sanity"); 4876 InstructionMark im(this); 4877 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 4878 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4879 attributes.set_embedded_opmask_register_specifier(mask); 4880 attributes.set_is_evex_instruction(); 4881 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4882 emit_int8(0x31); 4883 emit_operand(dst, src, 0); 4884 } 4885 4886 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) { 4887 evpmovzxbd(dst, k0, src, vector_len); 4888 } 4889 4890 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 4891 assert(VM_Version::supports_evex(), ""); 4892 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r 4893 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4894 attributes.set_is_evex_instruction(); 4895 attributes.set_embedded_opmask_register_specifier(mask); 4896 if (merge) { 4897 attributes.reset_is_clear_context(); 4898 } 4899 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 4900 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 4901 } 4902 4903 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) { 4904 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 4905 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4906 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4907 emit_int16(0x35, (0xC0 | encode)); 4908 } 4909 4910 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) { 4911 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 4912 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4913 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4914 emit_int16(0x31, (0xC0 | encode)); 4915 } 4916 4917 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) { 4918 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), ""); 4919 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4920 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4921 emit_int16(0x32, (0xC0 | encode)); 4922 } 4923 4924 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) { 4925 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 4926 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 4927 VM_Version::supports_evex(), ""); 4928 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4929 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4930 emit_int16(0x21, (0xC0 | encode)); 4931 } 4932 4933 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) { 4934 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 4935 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 4936 VM_Version::supports_evex(), ""); 4937 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4938 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4939 emit_int16(0x22, (0xC0 | encode)); 4940 } 4941 4942 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) { 4943 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 4944 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 4945 VM_Version::supports_evex(), ""); 4946 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4947 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4948 emit_int16(0x23, (0xC0 | encode)); 4949 } 4950 4951 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) { 4952 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 4953 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 4954 VM_Version::supports_evex(), ""); 4955 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4956 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4957 emit_int16(0x24, (0xC0 | encode)); 4958 } 4959 4960 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) { 4961 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 4962 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 4963 VM_Version::supports_evex(), ""); 4964 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4965 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 4966 emit_int16(0x25, (0xC0 | encode)); 4967 } 4968 4969 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) { 4970 assert(VM_Version::supports_avx512vlbw(), ""); 4971 assert(src != xnoreg, "sanity"); 4972 InstructionMark im(this); 4973 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 4974 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4975 attributes.set_is_evex_instruction(); 4976 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 4977 emit_int8(0x30); 4978 emit_operand(src, dst, 0); 4979 } 4980 4981 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) { 4982 assert(VM_Version::supports_avx512vlbw(), ""); 4983 assert(src != xnoreg, "sanity"); 4984 InstructionMark im(this); 4985 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 4986 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit); 4987 attributes.reset_is_clear_context(); 4988 attributes.set_embedded_opmask_register_specifier(mask); 4989 attributes.set_is_evex_instruction(); 4990 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 4991 emit_int8(0x30); 4992 emit_operand(src, dst, 0); 4993 } 4994 4995 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) { 4996 assert(VM_Version::supports_evex(), ""); 4997 assert(src != xnoreg, "sanity"); 4998 InstructionMark im(this); 4999 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5000 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit); 5001 attributes.set_is_evex_instruction(); 5002 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 5003 emit_int8(0x31); 5004 emit_operand(src, dst, 0); 5005 } 5006 5007 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) { 5008 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5009 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5010 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " "); 5011 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5012 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5013 emit_int16(0x33, (0xC0 | encode)); 5014 } 5015 5016 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) { 5017 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5018 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5019 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " "); 5020 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5021 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5022 emit_int16(0x34, (0xC0 | encode)); 5023 } 5024 5025 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) { 5026 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5027 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5028 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5029 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 5030 } 5031 5032 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5033 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5034 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5035 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), ""); 5036 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5037 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5038 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 5039 } 5040 5041 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5042 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5043 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5044 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5045 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5046 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5047 emit_int16(0x04, (0xC0 | encode)); 5048 } 5049 5050 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5051 evpmadd52luq(dst, k0, src1, src2, false, vector_len); 5052 } 5053 5054 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5055 assert(VM_Version::supports_avx512ifma(), ""); 5056 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5057 attributes.set_is_evex_instruction(); 5058 attributes.set_embedded_opmask_register_specifier(mask); 5059 if (merge) { 5060 attributes.reset_is_clear_context(); 5061 } 5062 5063 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5064 emit_int16((unsigned char)0xB4, (0xC0 | encode)); 5065 } 5066 5067 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5068 evpmadd52huq(dst, k0, src1, src2, false, vector_len); 5069 } 5070 5071 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5072 assert(VM_Version::supports_avx512ifma(), ""); 5073 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5074 attributes.set_is_evex_instruction(); 5075 attributes.set_embedded_opmask_register_specifier(mask); 5076 if (merge) { 5077 attributes.reset_is_clear_context(); 5078 } 5079 5080 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5081 emit_int16((unsigned char)0xB5, (0xC0 | encode)); 5082 } 5083 5084 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5085 assert(VM_Version::supports_evex(), ""); 5086 assert(VM_Version::supports_avx512_vnni(), "must support vnni"); 5087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5088 attributes.set_is_evex_instruction(); 5089 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5090 emit_int16(0x52, (0xC0 | encode)); 5091 } 5092 5093 // generic 5094 void Assembler::pop(Register dst) { 5095 int encode = prefix_and_encode(dst->encoding()); 5096 emit_int8(0x58 | encode); 5097 } 5098 5099 void Assembler::popcntl(Register dst, Address src) { 5100 assert(VM_Version::supports_popcnt(), "must support"); 5101 InstructionMark im(this); 5102 emit_int8((unsigned char)0xF3); 5103 prefix(src, dst); 5104 emit_int16(0x0F, (unsigned char)0xB8); 5105 emit_operand(dst, src, 0); 5106 } 5107 5108 void Assembler::popcntl(Register dst, Register src) { 5109 assert(VM_Version::supports_popcnt(), "must support"); 5110 emit_int8((unsigned char)0xF3); 5111 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 5112 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode)); 5113 } 5114 5115 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5116 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature"); 5117 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5118 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5119 attributes.set_embedded_opmask_register_specifier(mask); 5120 attributes.set_is_evex_instruction(); 5121 if (merge) { 5122 attributes.reset_is_clear_context(); 5123 } 5124 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5125 emit_int16(0x54, (0xC0 | encode)); 5126 } 5127 5128 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5129 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature"); 5130 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5131 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5132 attributes.set_is_evex_instruction(); 5133 attributes.set_embedded_opmask_register_specifier(mask); 5134 if (merge) { 5135 attributes.reset_is_clear_context(); 5136 } 5137 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5138 emit_int16(0x54, (0xC0 | encode)); 5139 } 5140 5141 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5142 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature"); 5143 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5144 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5145 attributes.set_is_evex_instruction(); 5146 attributes.set_embedded_opmask_register_specifier(mask); 5147 if (merge) { 5148 attributes.reset_is_clear_context(); 5149 } 5150 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5151 emit_int16(0x55, (0xC0 | encode)); 5152 } 5153 5154 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 5155 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature"); 5156 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 5157 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5158 attributes.set_is_evex_instruction(); 5159 attributes.set_embedded_opmask_register_specifier(mask); 5160 if (merge) { 5161 attributes.reset_is_clear_context(); 5162 } 5163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5164 emit_int16(0x55, (0xC0 | encode)); 5165 } 5166 5167 void Assembler::popf() { 5168 emit_int8((unsigned char)0x9D); 5169 } 5170 5171 #ifndef _LP64 // no 32bit push/pop on amd64 5172 void Assembler::popl(Address dst) { 5173 // NOTE: this will adjust stack by 8byte on 64bits 5174 InstructionMark im(this); 5175 prefix(dst); 5176 emit_int8((unsigned char)0x8F); 5177 emit_operand(rax, dst, 0); 5178 } 5179 #endif 5180 5181 void Assembler::prefetchnta(Address src) { 5182 NOT_LP64(assert(VM_Version::supports_sse(), "must support")); 5183 InstructionMark im(this); 5184 prefix(src); 5185 emit_int16(0x0F, 0x18); 5186 emit_operand(rax, src, 0); // 0, src 5187 } 5188 5189 void Assembler::prefetchr(Address src) { 5190 assert(VM_Version::supports_3dnow_prefetch(), "must support"); 5191 InstructionMark im(this); 5192 prefix(src); 5193 emit_int16(0x0F, 0x0D); 5194 emit_operand(rax, src, 0); // 0, src 5195 } 5196 5197 void Assembler::prefetcht0(Address src) { 5198 NOT_LP64(assert(VM_Version::supports_sse(), "must support")); 5199 InstructionMark im(this); 5200 prefix(src); 5201 emit_int16(0x0F, 0x18); 5202 emit_operand(rcx, src, 0); // 1, src 5203 } 5204 5205 void Assembler::prefetcht1(Address src) { 5206 NOT_LP64(assert(VM_Version::supports_sse(), "must support")); 5207 InstructionMark im(this); 5208 prefix(src); 5209 emit_int16(0x0F, 0x18); 5210 emit_operand(rdx, src, 0); // 2, src 5211 } 5212 5213 void Assembler::prefetcht2(Address src) { 5214 NOT_LP64(assert(VM_Version::supports_sse(), "must support")); 5215 InstructionMark im(this); 5216 prefix(src); 5217 emit_int16(0x0F, 0x18); 5218 emit_operand(rbx, src, 0); // 3, src 5219 } 5220 5221 void Assembler::prefetchw(Address src) { 5222 assert(VM_Version::supports_3dnow_prefetch(), "must support"); 5223 InstructionMark im(this); 5224 prefix(src); 5225 emit_int16(0x0F, 0x0D); 5226 emit_operand(rcx, src, 0); // 1, src 5227 } 5228 5229 void Assembler::prefix(Prefix p) { 5230 emit_int8(p); 5231 } 5232 5233 void Assembler::pshufb(XMMRegister dst, XMMRegister src) { 5234 assert(VM_Version::supports_ssse3(), ""); 5235 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5236 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5237 emit_int16(0x00, (0xC0 | encode)); 5238 } 5239 5240 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 5241 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 5242 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5243 attributes.set_is_evex_instruction(); 5244 attributes.set_embedded_opmask_register_specifier(mask); 5245 if (merge) { 5246 attributes.reset_is_clear_context(); 5247 } 5248 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5249 emit_int16(0x00, (0xC0 | encode)); 5250 } 5251 5252 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5253 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5254 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5255 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, ""); 5256 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5257 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5258 emit_int16(0x00, (0xC0 | encode)); 5259 } 5260 5261 void Assembler::pshufb(XMMRegister dst, Address src) { 5262 assert(VM_Version::supports_ssse3(), ""); 5263 InstructionMark im(this); 5264 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5265 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5266 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5267 emit_int8(0x00); 5268 emit_operand(dst, src, 0); 5269 } 5270 5271 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) { 5272 assert(isByte(mode), "invalid value"); 5273 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5274 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit; 5275 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5276 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5277 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5278 } 5279 5280 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5281 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5282 (vector_len == AVX_256bit? VM_Version::supports_avx2() : 5283 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), ""); 5284 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5285 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5286 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5287 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5288 } 5289 5290 void Assembler::pshufd(XMMRegister dst, Address src, int mode) { 5291 assert(isByte(mode), "invalid value"); 5292 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5293 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5294 InstructionMark im(this); 5295 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5296 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 5297 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5298 emit_int8(0x70); 5299 emit_operand(dst, src, 1); 5300 emit_int8(mode & 0xFF); 5301 } 5302 5303 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) { 5304 assert(isByte(mode), "invalid value"); 5305 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5306 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5307 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 5308 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5309 } 5310 5311 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5312 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5313 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5314 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), ""); 5315 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5316 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5317 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 5318 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5319 } 5320 5321 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 5322 assert(isByte(mode), "invalid value"); 5323 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5324 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5325 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5326 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5327 } 5328 5329 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) { 5330 assert(isByte(mode), "invalid value"); 5331 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5332 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5333 InstructionMark im(this); 5334 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5335 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5336 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5337 emit_int8(0x70); 5338 emit_operand(dst, src, 1); 5339 emit_int8(mode & 0xFF); 5340 } 5341 5342 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) { 5343 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5344 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5345 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), ""); 5346 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5347 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5348 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 5349 emit_int24(0x70, (0xC0 | encode), mode & 0xFF); 5350 } 5351 5352 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 5353 assert(VM_Version::supports_evex(), "requires EVEX support"); 5354 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, ""); 5355 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5356 attributes.set_is_evex_instruction(); 5357 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5358 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF); 5359 } 5360 5361 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) { 5362 assert(isByte(imm8), "invalid value"); 5363 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5364 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5366 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 5367 } 5368 5369 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 5370 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5371 attributes.set_rex_vex_w_reverted(); 5372 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5373 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 5374 } 5375 5376 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) { 5377 assert(isByte(imm8), "invalid value"); 5378 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5379 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5380 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 5381 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 5382 } 5383 5384 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 5385 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5386 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 5387 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF); 5388 } 5389 5390 void Assembler::psrldq(XMMRegister dst, int shift) { 5391 // Shift left 128 bit value in dst XMMRegister by shift number of bytes. 5392 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5393 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5394 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5395 emit_int24(0x73, (0xC0 | encode), shift); 5396 } 5397 5398 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 5399 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5400 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5401 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, ""); 5402 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5403 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5404 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 5405 } 5406 5407 void Assembler::pslldq(XMMRegister dst, int shift) { 5408 // Shift left 128 bit value in dst XMMRegister by shift number of bytes. 5409 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5410 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5411 // XMM7 is for /7 encoding: 66 0F 73 /7 ib 5412 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5413 emit_int24(0x73, (0xC0 | encode), shift); 5414 } 5415 5416 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 5417 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 5418 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 5419 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, ""); 5420 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5421 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5422 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 5423 } 5424 5425 void Assembler::ptest(XMMRegister dst, Address src) { 5426 assert(VM_Version::supports_sse4_1(), ""); 5427 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5428 InstructionMark im(this); 5429 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5430 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5431 emit_int8(0x17); 5432 emit_operand(dst, src, 0); 5433 } 5434 5435 void Assembler::ptest(XMMRegister dst, XMMRegister src) { 5436 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), ""); 5437 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5438 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5439 emit_int8(0x17); 5440 emit_int8((0xC0 | encode)); 5441 } 5442 5443 void Assembler::vptest(XMMRegister dst, Address src) { 5444 assert(VM_Version::supports_avx(), ""); 5445 InstructionMark im(this); 5446 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5447 assert(dst != xnoreg, "sanity"); 5448 // swap src<->dst for encoding 5449 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5450 emit_int8(0x17); 5451 emit_operand(dst, src, 0); 5452 } 5453 5454 void Assembler::vptest(XMMRegister dst, XMMRegister src) { 5455 assert(VM_Version::supports_avx(), ""); 5456 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5457 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5458 emit_int16(0x17, (0xC0 | encode)); 5459 } 5460 5461 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) { 5462 assert(VM_Version::supports_avx(), ""); 5463 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5464 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5465 emit_int16(0x17, (0xC0 | encode)); 5466 } 5467 5468 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) { 5469 assert(VM_Version::supports_avx(), ""); 5470 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5471 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5472 emit_int16(0x0E, (0xC0 | encode)); 5473 } 5474 5475 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5476 assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), ""); 5477 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r 5478 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5479 attributes.set_is_evex_instruction(); 5480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5481 emit_int16(0x26, (0xC0 | encode)); 5482 } 5483 5484 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5485 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), ""); 5486 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r 5487 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5488 attributes.set_is_evex_instruction(); 5489 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 5490 emit_int16(0x27, (0xC0 | encode)); 5491 } 5492 5493 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 5494 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), ""); 5495 // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r 5496 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5497 attributes.set_is_evex_instruction(); 5498 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 5499 emit_int16(0x27, (0xC0 | encode)); 5500 } 5501 5502 void Assembler::punpcklbw(XMMRegister dst, Address src) { 5503 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5504 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5505 InstructionMark im(this); 5506 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true); 5507 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 5508 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5509 emit_int8(0x60); 5510 emit_operand(dst, src, 0); 5511 } 5512 5513 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) { 5514 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true); 5516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5517 emit_int16(0x60, (0xC0 | encode)); 5518 } 5519 5520 void Assembler::punpckldq(XMMRegister dst, Address src) { 5521 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5522 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes"); 5523 InstructionMark im(this); 5524 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5525 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 5526 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5527 emit_int8(0x62); 5528 emit_operand(dst, src, 0); 5529 } 5530 5531 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 5532 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5533 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5534 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5535 emit_int16(0x62, (0xC0 | encode)); 5536 } 5537 5538 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) { 5539 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 5540 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5541 attributes.set_rex_vex_w_reverted(); 5542 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5543 emit_int16(0x6C, (0xC0 | encode)); 5544 } 5545 5546 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5547 evpunpcklqdq(dst, k0, src1, src2, false, vector_len); 5548 } 5549 5550 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5551 assert(VM_Version::supports_evex(), "requires AVX512F"); 5552 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 5553 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5554 attributes.set_is_evex_instruction(); 5555 attributes.set_embedded_opmask_register_specifier(mask); 5556 if (merge) { 5557 attributes.reset_is_clear_context(); 5558 } 5559 5560 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5561 emit_int16(0x6C, (0xC0 | encode)); 5562 } 5563 5564 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 5565 evpunpckhqdq(dst, k0, src1, src2, false, vector_len); 5566 } 5567 5568 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) { 5569 assert(VM_Version::supports_evex(), "requires AVX512F"); 5570 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 5571 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 5572 attributes.set_is_evex_instruction(); 5573 attributes.set_embedded_opmask_register_specifier(mask); 5574 if (merge) { 5575 attributes.reset_is_clear_context(); 5576 } 5577 5578 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 5579 emit_int16(0x6D, (0xC0 | encode)); 5580 } 5581 5582 void Assembler::push(int32_t imm32) { 5583 // in 64bits we push 64bits onto the stack but only 5584 // take a 32bit immediate 5585 emit_int8(0x68); 5586 emit_int32(imm32); 5587 } 5588 5589 void Assembler::push(Register src) { 5590 int encode = prefix_and_encode(src->encoding()); 5591 emit_int8(0x50 | encode); 5592 } 5593 5594 void Assembler::pushf() { 5595 emit_int8((unsigned char)0x9C); 5596 } 5597 5598 #ifndef _LP64 // no 32bit push/pop on amd64 5599 void Assembler::pushl(Address src) { 5600 // Note this will push 64bit on 64bit 5601 InstructionMark im(this); 5602 prefix(src); 5603 emit_int8((unsigned char)0xFF); 5604 emit_operand(rsi, src, 0); 5605 } 5606 #endif 5607 5608 void Assembler::rcll(Register dst, int imm8) { 5609 assert(isShiftCount(imm8), "illegal shift count"); 5610 int encode = prefix_and_encode(dst->encoding()); 5611 if (imm8 == 1) { 5612 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 5613 } else { 5614 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 5615 } 5616 } 5617 5618 void Assembler::rcpps(XMMRegister dst, XMMRegister src) { 5619 NOT_LP64(assert(VM_Version::supports_sse(), "")); 5620 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5621 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 5622 emit_int16(0x53, (0xC0 | encode)); 5623 } 5624 5625 void Assembler::rcpss(XMMRegister dst, XMMRegister src) { 5626 NOT_LP64(assert(VM_Version::supports_sse(), "")); 5627 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5628 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 5629 emit_int16(0x53, (0xC0 | encode)); 5630 } 5631 5632 void Assembler::rdtsc() { 5633 emit_int16(0x0F, 0x31); 5634 } 5635 5636 void Assembler::rdtscp() { 5637 emit_int24(0x0F, 0x01, (unsigned char)0xF9); 5638 } 5639 5640 // copies data from [esi] to [edi] using rcx pointer sized words 5641 // generic 5642 void Assembler::rep_mov() { 5643 // REP 5644 // MOVSQ 5645 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);) 5646 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xA5);) 5647 } 5648 5649 // sets rcx bytes with rax, value at [edi] 5650 void Assembler::rep_stosb() { 5651 // REP 5652 // STOSB 5653 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);) 5654 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAA);) 5655 } 5656 5657 // sets rcx pointer sized words with rax, value at [edi] 5658 // generic 5659 void Assembler::rep_stos() { 5660 // REP 5661 // LP64:STOSQ, LP32:STOSD 5662 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);) 5663 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAB);) 5664 } 5665 5666 // scans rcx pointer sized words at [edi] for occurrence of rax, 5667 // generic 5668 void Assembler::repne_scan() { // repne_scan 5669 // SCASQ 5670 LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);) 5671 NOT_LP64( emit_int16((unsigned char)0xF2, (unsigned char)0xAF);) 5672 } 5673 5674 #ifdef _LP64 5675 // scans rcx 4 byte words at [edi] for occurrence of rax, 5676 // generic 5677 void Assembler::repne_scanl() { // repne_scan 5678 // SCASL 5679 emit_int16((unsigned char)0xF2, (unsigned char)0xAF); 5680 } 5681 #endif 5682 5683 void Assembler::ret(int imm16) { 5684 if (imm16 == 0) { 5685 emit_int8((unsigned char)0xC3); 5686 } else { 5687 emit_int8((unsigned char)0xC2); 5688 emit_int16(imm16); 5689 } 5690 } 5691 5692 void Assembler::roll(Register dst, int imm8) { 5693 assert(isShiftCount(imm8), "illegal shift count"); 5694 int encode = prefix_and_encode(dst->encoding()); 5695 if (imm8 == 1) { 5696 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 5697 } else { 5698 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 5699 } 5700 } 5701 5702 void Assembler::roll(Register dst) { 5703 int encode = prefix_and_encode(dst->encoding()); 5704 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 5705 } 5706 5707 void Assembler::rorl(Register dst, int imm8) { 5708 assert(isShiftCount(imm8), "illegal shift count"); 5709 int encode = prefix_and_encode(dst->encoding()); 5710 if (imm8 == 1) { 5711 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 5712 } else { 5713 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8); 5714 } 5715 } 5716 5717 void Assembler::rorl(Register dst) { 5718 int encode = prefix_and_encode(dst->encoding()); 5719 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 5720 } 5721 5722 #ifdef _LP64 5723 void Assembler::rorq(Register dst) { 5724 int encode = prefixq_and_encode(dst->encoding()); 5725 emit_int16((unsigned char)0xD3, (0xC8 | encode)); 5726 } 5727 5728 void Assembler::rorq(Register dst, int imm8) { 5729 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 5730 int encode = prefixq_and_encode(dst->encoding()); 5731 if (imm8 == 1) { 5732 emit_int16((unsigned char)0xD1, (0xC8 | encode)); 5733 } else { 5734 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8); 5735 } 5736 } 5737 5738 void Assembler::rolq(Register dst) { 5739 int encode = prefixq_and_encode(dst->encoding()); 5740 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 5741 } 5742 5743 void Assembler::rolq(Register dst, int imm8) { 5744 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 5745 int encode = prefixq_and_encode(dst->encoding()); 5746 if (imm8 == 1) { 5747 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 5748 } else { 5749 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8); 5750 } 5751 } 5752 #endif 5753 5754 void Assembler::sahf() { 5755 #ifdef _LP64 5756 // Not supported in 64bit mode 5757 ShouldNotReachHere(); 5758 #endif 5759 emit_int8((unsigned char)0x9E); 5760 } 5761 5762 void Assembler::sall(Address dst, int imm8) { 5763 InstructionMark im(this); 5764 assert(isShiftCount(imm8), "illegal shift count"); 5765 prefix(dst); 5766 if (imm8 == 1) { 5767 emit_int8((unsigned char)0xD1); 5768 emit_operand(as_Register(4), dst, 0); 5769 } 5770 else { 5771 emit_int8((unsigned char)0xC1); 5772 emit_operand(as_Register(4), dst, 1); 5773 emit_int8(imm8); 5774 } 5775 } 5776 5777 void Assembler::sall(Address dst) { 5778 InstructionMark im(this); 5779 prefix(dst); 5780 emit_int8((unsigned char)0xD3); 5781 emit_operand(as_Register(4), dst, 0); 5782 } 5783 5784 void Assembler::sall(Register dst, int imm8) { 5785 assert(isShiftCount(imm8), "illegal shift count"); 5786 int encode = prefix_and_encode(dst->encoding()); 5787 if (imm8 == 1) { 5788 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 5789 } else { 5790 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 5791 } 5792 } 5793 5794 void Assembler::sall(Register dst) { 5795 int encode = prefix_and_encode(dst->encoding()); 5796 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 5797 } 5798 5799 void Assembler::sarl(Address dst, int imm8) { 5800 assert(isShiftCount(imm8), "illegal shift count"); 5801 InstructionMark im(this); 5802 prefix(dst); 5803 if (imm8 == 1) { 5804 emit_int8((unsigned char)0xD1); 5805 emit_operand(as_Register(7), dst, 0); 5806 } 5807 else { 5808 emit_int8((unsigned char)0xC1); 5809 emit_operand(as_Register(7), dst, 1); 5810 emit_int8(imm8); 5811 } 5812 } 5813 5814 void Assembler::sarl(Address dst) { 5815 InstructionMark im(this); 5816 prefix(dst); 5817 emit_int8((unsigned char)0xD3); 5818 emit_operand(as_Register(7), dst, 0); 5819 } 5820 5821 void Assembler::sarl(Register dst, int imm8) { 5822 int encode = prefix_and_encode(dst->encoding()); 5823 assert(isShiftCount(imm8), "illegal shift count"); 5824 if (imm8 == 1) { 5825 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 5826 } else { 5827 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 5828 } 5829 } 5830 5831 void Assembler::sarl(Register dst) { 5832 int encode = prefix_and_encode(dst->encoding()); 5833 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 5834 } 5835 5836 void Assembler::sbbl(Address dst, int32_t imm32) { 5837 InstructionMark im(this); 5838 prefix(dst); 5839 emit_arith_operand(0x81, rbx, dst, imm32); 5840 } 5841 5842 void Assembler::sbbl(Register dst, int32_t imm32) { 5843 prefix(dst); 5844 emit_arith(0x81, 0xD8, dst, imm32); 5845 } 5846 5847 5848 void Assembler::sbbl(Register dst, Address src) { 5849 InstructionMark im(this); 5850 prefix(src, dst); 5851 emit_int8(0x1B); 5852 emit_operand(dst, src, 0); 5853 } 5854 5855 void Assembler::sbbl(Register dst, Register src) { 5856 (void) prefix_and_encode(dst->encoding(), src->encoding()); 5857 emit_arith(0x1B, 0xC0, dst, src); 5858 } 5859 5860 void Assembler::setb(Condition cc, Register dst) { 5861 assert(0 <= cc && cc < 16, "illegal cc"); 5862 int encode = prefix_and_encode(dst->encoding(), true); 5863 emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode)); 5864 } 5865 5866 void Assembler::sete(Register dst) { 5867 int encode = prefix_and_encode(dst->encoding(), true); 5868 emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode)); 5869 } 5870 5871 void Assembler::setl(Register dst) { 5872 int encode = prefix_and_encode(dst->encoding(), true); 5873 emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode)); 5874 } 5875 5876 void Assembler::setne(Register dst) { 5877 int encode = prefix_and_encode(dst->encoding(), true); 5878 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode)); 5879 } 5880 5881 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) { 5882 assert(VM_Version::supports_ssse3(), ""); 5883 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5884 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5885 emit_int24(0x0F, (0xC0 | encode), imm8); 5886 } 5887 5888 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 5889 assert(vector_len == AVX_128bit? VM_Version::supports_avx() : 5890 vector_len == AVX_256bit? VM_Version::supports_avx2() : 5891 0, ""); 5892 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 5893 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5894 emit_int24(0x0F, (0xC0 | encode), imm8); 5895 } 5896 5897 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 5898 assert(VM_Version::supports_evex(), ""); 5899 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 5900 attributes.set_is_evex_instruction(); 5901 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5902 emit_int24(0x3, (0xC0 | encode), imm8); 5903 } 5904 5905 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) { 5906 assert(VM_Version::supports_sse4_1(), ""); 5907 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 5908 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 5909 emit_int24(0x0E, (0xC0 | encode), imm8); 5910 } 5911 5912 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) { 5913 assert(VM_Version::supports_sha(), ""); 5914 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false); 5915 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8); 5916 } 5917 5918 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) { 5919 assert(VM_Version::supports_sha(), ""); 5920 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5921 emit_int16((unsigned char)0xC8, (0xC0 | encode)); 5922 } 5923 5924 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) { 5925 assert(VM_Version::supports_sha(), ""); 5926 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5927 emit_int16((unsigned char)0xC9, (0xC0 | encode)); 5928 } 5929 5930 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) { 5931 assert(VM_Version::supports_sha(), ""); 5932 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5933 emit_int16((unsigned char)0xCA, (0xC0 | encode)); 5934 } 5935 5936 // xmm0 is implicit additional source to this instruction. 5937 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) { 5938 assert(VM_Version::supports_sha(), ""); 5939 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5940 emit_int16((unsigned char)0xCB, (0xC0 | encode)); 5941 } 5942 5943 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) { 5944 assert(VM_Version::supports_sha(), ""); 5945 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5946 emit_int16((unsigned char)0xCC, (0xC0 | encode)); 5947 } 5948 5949 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) { 5950 assert(VM_Version::supports_sha(), ""); 5951 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false); 5952 emit_int16((unsigned char)0xCD, (0xC0 | encode)); 5953 } 5954 5955 5956 void Assembler::shll(Register dst, int imm8) { 5957 assert(isShiftCount(imm8), "illegal shift count"); 5958 int encode = prefix_and_encode(dst->encoding()); 5959 if (imm8 == 1 ) { 5960 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 5961 } else { 5962 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 5963 } 5964 } 5965 5966 void Assembler::shll(Register dst) { 5967 int encode = prefix_and_encode(dst->encoding()); 5968 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 5969 } 5970 5971 void Assembler::shrl(Register dst, int imm8) { 5972 assert(isShiftCount(imm8), "illegal shift count"); 5973 int encode = prefix_and_encode(dst->encoding()); 5974 if (imm8 == 1) { 5975 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 5976 } 5977 else { 5978 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 5979 } 5980 } 5981 5982 void Assembler::shrl(Register dst) { 5983 int encode = prefix_and_encode(dst->encoding()); 5984 emit_int16((unsigned char)0xD3, (0xE8 | encode)); 5985 } 5986 5987 void Assembler::shrl(Address dst) { 5988 InstructionMark im(this); 5989 prefix(dst); 5990 emit_int8((unsigned char)0xD3); 5991 emit_operand(as_Register(5), dst, 0); 5992 } 5993 5994 void Assembler::shrl(Address dst, int imm8) { 5995 InstructionMark im(this); 5996 assert(isShiftCount(imm8), "illegal shift count"); 5997 prefix(dst); 5998 if (imm8 == 1) { 5999 emit_int8((unsigned char)0xD1); 6000 emit_operand(as_Register(5), dst, 0); 6001 } 6002 else { 6003 emit_int8((unsigned char)0xC1); 6004 emit_operand(as_Register(5), dst, 1); 6005 emit_int8(imm8); 6006 } 6007 } 6008 6009 6010 void Assembler::shldl(Register dst, Register src) { 6011 int encode = prefix_and_encode(src->encoding(), dst->encoding()); 6012 emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode)); 6013 } 6014 6015 void Assembler::shldl(Register dst, Register src, int8_t imm8) { 6016 int encode = prefix_and_encode(src->encoding(), dst->encoding()); 6017 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8); 6018 } 6019 6020 void Assembler::shrdl(Register dst, Register src) { 6021 int encode = prefix_and_encode(src->encoding(), dst->encoding()); 6022 emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode)); 6023 } 6024 6025 void Assembler::shrdl(Register dst, Register src, int8_t imm8) { 6026 int encode = prefix_and_encode(src->encoding(), dst->encoding()); 6027 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8); 6028 } 6029 6030 #ifdef _LP64 6031 void Assembler::shldq(Register dst, Register src, int8_t imm8) { 6032 int encode = prefixq_and_encode(src->encoding(), dst->encoding()); 6033 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8); 6034 } 6035 6036 void Assembler::shrdq(Register dst, Register src, int8_t imm8) { 6037 int encode = prefixq_and_encode(src->encoding(), dst->encoding()); 6038 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8); 6039 } 6040 #endif 6041 6042 // copies a single word from [esi] to [edi] 6043 void Assembler::smovl() { 6044 emit_int8((unsigned char)0xA5); 6045 } 6046 6047 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) { 6048 assert(VM_Version::supports_sse4_1(), ""); 6049 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6050 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6051 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode); 6052 } 6053 6054 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) { 6055 assert(VM_Version::supports_sse4_1(), ""); 6056 InstructionMark im(this); 6057 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6058 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6059 emit_int8(0x0B); 6060 emit_operand(dst, src, 1); 6061 emit_int8((unsigned char)rmode); 6062 } 6063 6064 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 6065 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6066 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6067 attributes.set_rex_vex_w_reverted(); 6068 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6069 emit_int16(0x51, (0xC0 | encode)); 6070 } 6071 6072 void Assembler::sqrtsd(XMMRegister dst, Address src) { 6073 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6074 InstructionMark im(this); 6075 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6076 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6077 attributes.set_rex_vex_w_reverted(); 6078 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6079 emit_int8(0x51); 6080 emit_operand(dst, src, 0); 6081 } 6082 6083 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 6084 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6085 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6086 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6087 emit_int16(0x51, (0xC0 | encode)); 6088 } 6089 6090 void Assembler::std() { 6091 emit_int8((unsigned char)0xFD); 6092 } 6093 6094 void Assembler::sqrtss(XMMRegister dst, Address src) { 6095 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6096 InstructionMark im(this); 6097 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6098 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6099 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6100 emit_int8(0x51); 6101 emit_operand(dst, src, 0); 6102 } 6103 6104 void Assembler::stmxcsr( Address dst) { 6105 if (UseAVX > 0 ) { 6106 assert(VM_Version::supports_avx(), ""); 6107 InstructionMark im(this); 6108 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6109 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6110 emit_int8((unsigned char)0xAE); 6111 emit_operand(as_Register(3), dst, 0); 6112 } else { 6113 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6114 InstructionMark im(this); 6115 prefix(dst); 6116 emit_int16(0x0F, (unsigned char)0xAE); 6117 emit_operand(as_Register(3), dst, 0); 6118 } 6119 } 6120 6121 void Assembler::subl(Address dst, int32_t imm32) { 6122 InstructionMark im(this); 6123 prefix(dst); 6124 emit_arith_operand(0x81, rbp, dst, imm32); 6125 } 6126 6127 void Assembler::subl(Address dst, Register src) { 6128 InstructionMark im(this); 6129 prefix(dst, src); 6130 emit_int8(0x29); 6131 emit_operand(src, dst, 0); 6132 } 6133 6134 void Assembler::subl(Register dst, int32_t imm32) { 6135 prefix(dst); 6136 emit_arith(0x81, 0xE8, dst, imm32); 6137 } 6138 6139 // Force generation of a 4 byte immediate value even if it fits into 8bit 6140 void Assembler::subl_imm32(Register dst, int32_t imm32) { 6141 prefix(dst); 6142 emit_arith_imm32(0x81, 0xE8, dst, imm32); 6143 } 6144 6145 void Assembler::subl(Register dst, Address src) { 6146 InstructionMark im(this); 6147 prefix(src, dst); 6148 emit_int8(0x2B); 6149 emit_operand(dst, src, 0); 6150 } 6151 6152 void Assembler::subl(Register dst, Register src) { 6153 (void) prefix_and_encode(dst->encoding(), src->encoding()); 6154 emit_arith(0x2B, 0xC0, dst, src); 6155 } 6156 6157 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 6158 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6159 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6160 attributes.set_rex_vex_w_reverted(); 6161 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6162 emit_int16(0x5C, (0xC0 | encode)); 6163 } 6164 6165 void Assembler::subsd(XMMRegister dst, Address src) { 6166 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6167 InstructionMark im(this); 6168 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6169 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6170 attributes.set_rex_vex_w_reverted(); 6171 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6172 emit_int8(0x5C); 6173 emit_operand(dst, src, 0); 6174 } 6175 6176 void Assembler::subss(XMMRegister dst, XMMRegister src) { 6177 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6178 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false); 6179 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6180 emit_int16(0x5C, (0xC0 | encode)); 6181 } 6182 6183 void Assembler::subss(XMMRegister dst, Address src) { 6184 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6185 InstructionMark im(this); 6186 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6187 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6188 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6189 emit_int8(0x5C); 6190 emit_operand(dst, src, 0); 6191 } 6192 6193 void Assembler::testb(Register dst, int imm8) { 6194 NOT_LP64(assert(dst->has_byte_register(), "must have byte register")); 6195 if (dst == rax) { 6196 emit_int8((unsigned char)0xA8); 6197 emit_int8(imm8); 6198 } else { 6199 (void) prefix_and_encode(dst->encoding(), true); 6200 emit_arith_b(0xF6, 0xC0, dst, imm8); 6201 } 6202 } 6203 6204 void Assembler::testb(Address dst, int imm8) { 6205 InstructionMark im(this); 6206 prefix(dst); 6207 emit_int8((unsigned char)0xF6); 6208 emit_operand(rax, dst, 1); 6209 emit_int8(imm8); 6210 } 6211 6212 void Assembler::testl(Address dst, int32_t imm32) { 6213 InstructionMark im(this); 6214 prefix(dst); 6215 emit_int8((unsigned char)0xF7); 6216 emit_operand(as_Register(0), dst, 4); 6217 emit_int32(imm32); 6218 } 6219 6220 void Assembler::testl(Register dst, int32_t imm32) { 6221 // not using emit_arith because test 6222 // doesn't support sign-extension of 6223 // 8bit operands 6224 if (dst == rax) { 6225 emit_int8((unsigned char)0xA9); 6226 emit_int32(imm32); 6227 } else { 6228 int encode = dst->encoding(); 6229 encode = prefix_and_encode(encode); 6230 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 6231 emit_int32(imm32); 6232 } 6233 } 6234 6235 void Assembler::testl(Register dst, Register src) { 6236 (void) prefix_and_encode(dst->encoding(), src->encoding()); 6237 emit_arith(0x85, 0xC0, dst, src); 6238 } 6239 6240 void Assembler::testl(Register dst, Address src) { 6241 InstructionMark im(this); 6242 prefix(src, dst); 6243 emit_int8((unsigned char)0x85); 6244 emit_operand(dst, src, 0); 6245 } 6246 6247 void Assembler::tzcntl(Register dst, Register src) { 6248 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 6249 emit_int8((unsigned char)0xF3); 6250 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 6251 emit_int24(0x0F, 6252 (unsigned char)0xBC, 6253 0xC0 | encode); 6254 } 6255 6256 void Assembler::tzcntl(Register dst, Address src) { 6257 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 6258 InstructionMark im(this); 6259 emit_int8((unsigned char)0xF3); 6260 prefix(src, dst); 6261 emit_int16(0x0F, (unsigned char)0xBC); 6262 emit_operand(dst, src, 0); 6263 } 6264 6265 void Assembler::tzcntq(Register dst, Register src) { 6266 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 6267 emit_int8((unsigned char)0xF3); 6268 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 6269 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode)); 6270 } 6271 6272 void Assembler::tzcntq(Register dst, Address src) { 6273 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); 6274 InstructionMark im(this); 6275 emit_int8((unsigned char)0xF3); 6276 prefixq(src, dst); 6277 emit_int16(0x0F, (unsigned char)0xBC); 6278 emit_operand(dst, src, 0); 6279 } 6280 6281 void Assembler::ucomisd(XMMRegister dst, Address src) { 6282 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6283 InstructionMark im(this); 6284 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6285 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6286 attributes.set_rex_vex_w_reverted(); 6287 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6288 emit_int8(0x2E); 6289 emit_operand(dst, src, 0); 6290 } 6291 6292 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 6293 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6294 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6295 attributes.set_rex_vex_w_reverted(); 6296 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6297 emit_int16(0x2E, (0xC0 | encode)); 6298 } 6299 6300 void Assembler::ucomiss(XMMRegister dst, Address src) { 6301 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6302 InstructionMark im(this); 6303 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6304 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6305 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6306 emit_int8(0x2E); 6307 emit_operand(dst, src, 0); 6308 } 6309 6310 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 6311 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6312 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6313 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6314 emit_int16(0x2E, (0xC0 | encode)); 6315 } 6316 6317 void Assembler::xabort(int8_t imm8) { 6318 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF)); 6319 } 6320 6321 void Assembler::xaddb(Address dst, Register src) { 6322 InstructionMark im(this); 6323 prefix(dst, src, true); 6324 emit_int16(0x0F, (unsigned char)0xC0); 6325 emit_operand(src, dst, 0); 6326 } 6327 6328 void Assembler::xaddw(Address dst, Register src) { 6329 InstructionMark im(this); 6330 emit_int8(0x66); 6331 prefix(dst, src); 6332 emit_int16(0x0F, (unsigned char)0xC1); 6333 emit_operand(src, dst, 0); 6334 } 6335 6336 void Assembler::xaddl(Address dst, Register src) { 6337 InstructionMark im(this); 6338 prefix(dst, src); 6339 emit_int16(0x0F, (unsigned char)0xC1); 6340 emit_operand(src, dst, 0); 6341 } 6342 6343 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) { 6344 InstructionMark im(this); 6345 relocate(rtype); 6346 if (abort.is_bound()) { 6347 address entry = target(abort); 6348 assert(entry != NULL, "abort entry NULL"); 6349 intptr_t offset = entry - pc(); 6350 emit_int16((unsigned char)0xC7, (unsigned char)0xF8); 6351 emit_int32(offset - 6); // 2 opcode + 4 address 6352 } else { 6353 abort.add_patch_at(code(), locator()); 6354 emit_int16((unsigned char)0xC7, (unsigned char)0xF8); 6355 emit_int32(0); 6356 } 6357 } 6358 6359 void Assembler::xchgb(Register dst, Address src) { // xchg 6360 InstructionMark im(this); 6361 prefix(src, dst, true); 6362 emit_int8((unsigned char)0x86); 6363 emit_operand(dst, src, 0); 6364 } 6365 6366 void Assembler::xchgw(Register dst, Address src) { // xchg 6367 InstructionMark im(this); 6368 emit_int8(0x66); 6369 prefix(src, dst); 6370 emit_int8((unsigned char)0x87); 6371 emit_operand(dst, src, 0); 6372 } 6373 6374 void Assembler::xchgl(Register dst, Address src) { // xchg 6375 InstructionMark im(this); 6376 prefix(src, dst); 6377 emit_int8((unsigned char)0x87); 6378 emit_operand(dst, src, 0); 6379 } 6380 6381 void Assembler::xchgl(Register dst, Register src) { 6382 int encode = prefix_and_encode(dst->encoding(), src->encoding()); 6383 emit_int16((unsigned char)0x87, (0xC0 | encode)); 6384 } 6385 6386 void Assembler::xend() { 6387 emit_int24(0x0F, 0x01, (unsigned char)0xD5); 6388 } 6389 6390 void Assembler::xgetbv() { 6391 emit_int24(0x0F, 0x01, (unsigned char)0xD0); 6392 } 6393 6394 void Assembler::xorl(Address dst, int32_t imm32) { 6395 InstructionMark im(this); 6396 prefix(dst); 6397 emit_arith_operand(0x81, as_Register(6), dst, imm32); 6398 } 6399 6400 void Assembler::xorl(Register dst, int32_t imm32) { 6401 prefix(dst); 6402 emit_arith(0x81, 0xF0, dst, imm32); 6403 } 6404 6405 void Assembler::xorl(Register dst, Address src) { 6406 InstructionMark im(this); 6407 prefix(src, dst); 6408 emit_int8(0x33); 6409 emit_operand(dst, src, 0); 6410 } 6411 6412 void Assembler::xorl(Register dst, Register src) { 6413 (void) prefix_and_encode(dst->encoding(), src->encoding()); 6414 emit_arith(0x33, 0xC0, dst, src); 6415 } 6416 6417 void Assembler::xorl(Address dst, Register src) { 6418 InstructionMark im(this); 6419 prefix(dst, src); 6420 emit_int8(0x31); 6421 emit_operand(src, dst, 0); 6422 } 6423 6424 void Assembler::xorb(Register dst, Address src) { 6425 InstructionMark im(this); 6426 prefix(src, dst); 6427 emit_int8(0x32); 6428 emit_operand(dst, src, 0); 6429 } 6430 6431 void Assembler::xorb(Address dst, Register src) { 6432 InstructionMark im(this); 6433 prefix(dst, src, true); 6434 emit_int8(0x30); 6435 emit_operand(src, dst, 0); 6436 } 6437 6438 void Assembler::xorw(Register dst, Register src) { 6439 (void)prefix_and_encode(dst->encoding(), src->encoding()); 6440 emit_arith(0x33, 0xC0, dst, src); 6441 } 6442 6443 // AVX 3-operands scalar float-point arithmetic instructions 6444 6445 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) { 6446 assert(VM_Version::supports_avx(), ""); 6447 InstructionMark im(this); 6448 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6449 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6450 attributes.set_rex_vex_w_reverted(); 6451 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6452 emit_int8(0x58); 6453 emit_operand(dst, src, 0); 6454 } 6455 6456 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6457 assert(VM_Version::supports_avx(), ""); 6458 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6459 attributes.set_rex_vex_w_reverted(); 6460 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6461 emit_int16(0x58, (0xC0 | encode)); 6462 } 6463 6464 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) { 6465 assert(VM_Version::supports_avx(), ""); 6466 InstructionMark im(this); 6467 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6468 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6469 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6470 emit_int8(0x58); 6471 emit_operand(dst, src, 0); 6472 } 6473 6474 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6475 assert(VM_Version::supports_avx(), ""); 6476 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6477 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6478 emit_int16(0x58, (0xC0 | encode)); 6479 } 6480 6481 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) { 6482 assert(VM_Version::supports_avx(), ""); 6483 InstructionMark im(this); 6484 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6485 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6486 attributes.set_rex_vex_w_reverted(); 6487 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6488 emit_int8(0x5E); 6489 emit_operand(dst, src, 0); 6490 } 6491 6492 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6493 assert(VM_Version::supports_avx(), ""); 6494 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6495 attributes.set_rex_vex_w_reverted(); 6496 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6497 emit_int16(0x5E, (0xC0 | encode)); 6498 } 6499 6500 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) { 6501 assert(VM_Version::supports_avx(), ""); 6502 InstructionMark im(this); 6503 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6504 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6505 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6506 emit_int8(0x5E); 6507 emit_operand(dst, src, 0); 6508 } 6509 6510 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6511 assert(VM_Version::supports_avx(), ""); 6512 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6513 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6514 emit_int16(0x5E, (0xC0 | encode)); 6515 } 6516 6517 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 6518 assert(VM_Version::supports_fma(), ""); 6519 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6520 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6521 emit_int16((unsigned char)0xB9, (0xC0 | encode)); 6522 } 6523 6524 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) { 6525 assert(VM_Version::supports_fma(), ""); 6526 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6527 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6528 emit_int16((unsigned char)0xB9, (0xC0 | encode)); 6529 } 6530 6531 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) { 6532 assert(VM_Version::supports_avx(), ""); 6533 InstructionMark im(this); 6534 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6535 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6536 attributes.set_rex_vex_w_reverted(); 6537 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6538 emit_int8(0x59); 6539 emit_operand(dst, src, 0); 6540 } 6541 6542 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6543 assert(VM_Version::supports_avx(), ""); 6544 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6545 attributes.set_rex_vex_w_reverted(); 6546 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6547 emit_int16(0x59, (0xC0 | encode)); 6548 } 6549 6550 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) { 6551 assert(VM_Version::supports_avx(), ""); 6552 InstructionMark im(this); 6553 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6554 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6555 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6556 emit_int8(0x59); 6557 emit_operand(dst, src, 0); 6558 } 6559 6560 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6561 assert(VM_Version::supports_avx(), ""); 6562 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6563 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6564 emit_int16(0x59, (0xC0 | encode)); 6565 } 6566 6567 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) { 6568 assert(VM_Version::supports_avx(), ""); 6569 InstructionMark im(this); 6570 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6571 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 6572 attributes.set_rex_vex_w_reverted(); 6573 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6574 emit_int8(0x5C); 6575 emit_operand(dst, src, 0); 6576 } 6577 6578 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6579 assert(VM_Version::supports_avx(), ""); 6580 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6581 attributes.set_rex_vex_w_reverted(); 6582 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 6583 emit_int16(0x5C, (0xC0 | encode)); 6584 } 6585 6586 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) { 6587 assert(VM_Version::supports_avx(), ""); 6588 InstructionMark im(this); 6589 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6590 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 6591 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6592 emit_int8(0x5C); 6593 emit_operand(dst, src, 0); 6594 } 6595 6596 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 6597 assert(VM_Version::supports_avx(), ""); 6598 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 6599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 6600 emit_int16(0x5C, (0xC0 | encode)); 6601 } 6602 6603 //====================VECTOR ARITHMETIC===================================== 6604 6605 // Float-point vector arithmetic 6606 6607 void Assembler::addpd(XMMRegister dst, XMMRegister src) { 6608 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6609 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6610 attributes.set_rex_vex_w_reverted(); 6611 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6612 emit_int16(0x58, (0xC0 | encode)); 6613 } 6614 6615 void Assembler::addpd(XMMRegister dst, Address src) { 6616 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6617 InstructionMark im(this); 6618 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6619 attributes.set_rex_vex_w_reverted(); 6620 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6621 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6622 emit_int8(0x58); 6623 emit_operand(dst, src, 0); 6624 } 6625 6626 6627 void Assembler::addps(XMMRegister dst, XMMRegister src) { 6628 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6629 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6630 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6631 emit_int16(0x58, (0xC0 | encode)); 6632 } 6633 6634 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6635 assert(VM_Version::supports_avx(), ""); 6636 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6637 attributes.set_rex_vex_w_reverted(); 6638 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6639 emit_int16(0x58, (0xC0 | encode)); 6640 } 6641 6642 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6643 assert(VM_Version::supports_avx(), ""); 6644 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6645 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6646 emit_int16(0x58, (0xC0 | encode)); 6647 } 6648 6649 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6650 assert(VM_Version::supports_avx(), ""); 6651 InstructionMark im(this); 6652 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6653 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6654 attributes.set_rex_vex_w_reverted(); 6655 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6656 emit_int8(0x58); 6657 emit_operand(dst, src, 0); 6658 } 6659 6660 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6661 assert(VM_Version::supports_avx(), ""); 6662 InstructionMark im(this); 6663 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6664 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6665 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6666 emit_int8(0x58); 6667 emit_operand(dst, src, 0); 6668 } 6669 6670 void Assembler::subpd(XMMRegister dst, XMMRegister src) { 6671 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6672 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6673 attributes.set_rex_vex_w_reverted(); 6674 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6675 emit_int16(0x5C, (0xC0 | encode)); 6676 } 6677 6678 void Assembler::subps(XMMRegister dst, XMMRegister src) { 6679 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6680 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6681 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6682 emit_int16(0x5C, (0xC0 | encode)); 6683 } 6684 6685 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6686 assert(VM_Version::supports_avx(), ""); 6687 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6688 attributes.set_rex_vex_w_reverted(); 6689 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6690 emit_int16(0x5C, (0xC0 | encode)); 6691 } 6692 6693 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6694 assert(VM_Version::supports_avx(), ""); 6695 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6696 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6697 emit_int16(0x5C, (0xC0 | encode)); 6698 } 6699 6700 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6701 assert(VM_Version::supports_avx(), ""); 6702 InstructionMark im(this); 6703 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6704 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6705 attributes.set_rex_vex_w_reverted(); 6706 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6707 emit_int8(0x5C); 6708 emit_operand(dst, src, 0); 6709 } 6710 6711 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6712 assert(VM_Version::supports_avx(), ""); 6713 InstructionMark im(this); 6714 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6715 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6716 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6717 emit_int8(0x5C); 6718 emit_operand(dst, src, 0); 6719 } 6720 6721 void Assembler::mulpd(XMMRegister dst, XMMRegister src) { 6722 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6723 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6724 attributes.set_rex_vex_w_reverted(); 6725 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6726 emit_int16(0x59, (0xC0 | encode)); 6727 } 6728 6729 void Assembler::mulpd(XMMRegister dst, Address src) { 6730 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6731 InstructionMark im(this); 6732 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6733 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6734 attributes.set_rex_vex_w_reverted(); 6735 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6736 emit_int8(0x59); 6737 emit_operand(dst, src, 0); 6738 } 6739 6740 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 6741 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6742 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6743 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6744 emit_int16(0x59, (0xC0 | encode)); 6745 } 6746 6747 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6748 assert(VM_Version::supports_avx(), ""); 6749 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6750 attributes.set_rex_vex_w_reverted(); 6751 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6752 emit_int16(0x59, (0xC0 | encode)); 6753 } 6754 6755 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6756 assert(VM_Version::supports_avx(), ""); 6757 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6758 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6759 emit_int16(0x59, (0xC0 | encode)); 6760 } 6761 6762 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6763 assert(VM_Version::supports_avx(), ""); 6764 InstructionMark im(this); 6765 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6766 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6767 attributes.set_rex_vex_w_reverted(); 6768 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6769 emit_int8(0x59); 6770 emit_operand(dst, src, 0); 6771 } 6772 6773 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6774 assert(VM_Version::supports_avx(), ""); 6775 InstructionMark im(this); 6776 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6777 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6778 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6779 emit_int8(0x59); 6780 emit_operand(dst, src, 0); 6781 } 6782 6783 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 6784 assert(VM_Version::supports_fma(), ""); 6785 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6786 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6787 emit_int16((unsigned char)0xB8, (0xC0 | encode)); 6788 } 6789 6790 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) { 6791 assert(VM_Version::supports_fma(), ""); 6792 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6793 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6794 emit_int16((unsigned char)0xB8, (0xC0 | encode)); 6795 } 6796 6797 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 6798 assert(VM_Version::supports_fma(), ""); 6799 InstructionMark im(this); 6800 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6801 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6802 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6803 emit_int8((unsigned char)0xB8); 6804 emit_operand(dst, src2, 0); 6805 } 6806 6807 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 6808 assert(VM_Version::supports_fma(), ""); 6809 InstructionMark im(this); 6810 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6811 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6812 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 6813 emit_int8((unsigned char)0xB8); 6814 emit_operand(dst, src2, 0); 6815 } 6816 6817 void Assembler::divpd(XMMRegister dst, XMMRegister src) { 6818 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6819 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6820 attributes.set_rex_vex_w_reverted(); 6821 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6822 emit_int16(0x5E, (0xC0 | encode)); 6823 } 6824 6825 void Assembler::divps(XMMRegister dst, XMMRegister src) { 6826 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6827 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6828 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6829 emit_int16(0x5E, (0xC0 | encode)); 6830 } 6831 6832 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6833 assert(VM_Version::supports_avx(), ""); 6834 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6835 attributes.set_rex_vex_w_reverted(); 6836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6837 emit_int16(0x5E, (0xC0 | encode)); 6838 } 6839 6840 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6841 assert(VM_Version::supports_avx(), ""); 6842 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6843 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6844 emit_int16(0x5E, (0xC0 | encode)); 6845 } 6846 6847 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6848 assert(VM_Version::supports_avx(), ""); 6849 InstructionMark im(this); 6850 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6851 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6852 attributes.set_rex_vex_w_reverted(); 6853 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6854 emit_int8(0x5E); 6855 emit_operand(dst, src, 0); 6856 } 6857 6858 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6859 assert(VM_Version::supports_avx(), ""); 6860 InstructionMark im(this); 6861 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6862 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6863 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6864 emit_int8(0x5E); 6865 emit_operand(dst, src, 0); 6866 } 6867 6868 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) { 6869 assert(VM_Version::supports_avx(), ""); 6870 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6871 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6872 emit_int24(0x09, (0xC0 | encode), (rmode)); 6873 } 6874 6875 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) { 6876 assert(VM_Version::supports_avx(), ""); 6877 InstructionMark im(this); 6878 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 6879 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6880 emit_int8(0x09); 6881 emit_operand(dst, src, 1); 6882 emit_int8((rmode)); 6883 } 6884 6885 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) { 6886 assert(VM_Version::supports_evex(), "requires EVEX support"); 6887 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6888 attributes.set_is_evex_instruction(); 6889 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6890 emit_int24(0x09, (0xC0 | encode), (rmode)); 6891 } 6892 6893 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) { 6894 assert(VM_Version::supports_evex(), "requires EVEX support"); 6895 assert(dst != xnoreg, "sanity"); 6896 InstructionMark im(this); 6897 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6898 attributes.set_is_evex_instruction(); 6899 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6900 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 6901 emit_int8(0x09); 6902 emit_operand(dst, src, 1); 6903 emit_int8((rmode)); 6904 } 6905 6906 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) { 6907 assert(VM_Version::supports_avx(), ""); 6908 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6909 attributes.set_rex_vex_w_reverted(); 6910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6911 emit_int16(0x51, (0xC0 | encode)); 6912 } 6913 6914 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) { 6915 assert(VM_Version::supports_avx(), ""); 6916 InstructionMark im(this); 6917 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6918 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6919 attributes.set_rex_vex_w_reverted(); 6920 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6921 emit_int8(0x51); 6922 emit_operand(dst, src, 0); 6923 } 6924 6925 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) { 6926 assert(VM_Version::supports_avx(), ""); 6927 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6928 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6929 emit_int16(0x51, (0xC0 | encode)); 6930 } 6931 6932 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) { 6933 assert(VM_Version::supports_avx(), ""); 6934 InstructionMark im(this); 6935 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 6936 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6937 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6938 emit_int8(0x51); 6939 emit_operand(dst, src, 0); 6940 } 6941 6942 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 6943 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6944 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6945 attributes.set_rex_vex_w_reverted(); 6946 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6947 emit_int16(0x54, (0xC0 | encode)); 6948 } 6949 6950 void Assembler::andps(XMMRegister dst, XMMRegister src) { 6951 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6952 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6953 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6954 emit_int16(0x54, (0xC0 | encode)); 6955 } 6956 6957 void Assembler::andps(XMMRegister dst, Address src) { 6958 NOT_LP64(assert(VM_Version::supports_sse(), "")); 6959 InstructionMark im(this); 6960 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6961 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 6962 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6963 emit_int8(0x54); 6964 emit_operand(dst, src, 0); 6965 } 6966 6967 void Assembler::andpd(XMMRegister dst, Address src) { 6968 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 6969 InstructionMark im(this); 6970 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6971 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6972 attributes.set_rex_vex_w_reverted(); 6973 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6974 emit_int8(0x54); 6975 emit_operand(dst, src, 0); 6976 } 6977 6978 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6979 assert(VM_Version::supports_avx(), ""); 6980 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6981 attributes.set_rex_vex_w_reverted(); 6982 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 6983 emit_int16(0x54, (0xC0 | encode)); 6984 } 6985 6986 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 6987 assert(VM_Version::supports_avx(), ""); 6988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6989 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 6990 emit_int16(0x54, (0xC0 | encode)); 6991 } 6992 6993 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 6994 assert(VM_Version::supports_avx(), ""); 6995 InstructionMark im(this); 6996 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 6997 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 6998 attributes.set_rex_vex_w_reverted(); 6999 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7000 emit_int8(0x54); 7001 emit_operand(dst, src, 0); 7002 } 7003 7004 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7005 assert(VM_Version::supports_avx(), ""); 7006 InstructionMark im(this); 7007 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7008 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7009 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7010 emit_int8(0x54); 7011 emit_operand(dst, src, 0); 7012 } 7013 7014 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) { 7015 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7016 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7017 attributes.set_rex_vex_w_reverted(); 7018 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7019 emit_int8(0x15); 7020 emit_int8((0xC0 | encode)); 7021 } 7022 7023 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) { 7024 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7025 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7026 attributes.set_rex_vex_w_reverted(); 7027 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7028 emit_int16(0x14, (0xC0 | encode)); 7029 } 7030 7031 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 7032 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7033 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7034 attributes.set_rex_vex_w_reverted(); 7035 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7036 emit_int16(0x57, (0xC0 | encode)); 7037 } 7038 7039 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 7040 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7041 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7042 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7043 emit_int16(0x57, (0xC0 | encode)); 7044 } 7045 7046 void Assembler::xorpd(XMMRegister dst, Address src) { 7047 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7048 InstructionMark im(this); 7049 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7050 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 7051 attributes.set_rex_vex_w_reverted(); 7052 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7053 emit_int8(0x57); 7054 emit_operand(dst, src, 0); 7055 } 7056 7057 void Assembler::xorps(XMMRegister dst, Address src) { 7058 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7059 InstructionMark im(this); 7060 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7061 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7062 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7063 emit_int8(0x57); 7064 emit_operand(dst, src, 0); 7065 } 7066 7067 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7068 assert(VM_Version::supports_avx(), ""); 7069 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7070 attributes.set_rex_vex_w_reverted(); 7071 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7072 emit_int16(0x57, (0xC0 | encode)); 7073 } 7074 7075 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7076 assert(VM_Version::supports_avx(), ""); 7077 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7078 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7079 emit_int16(0x57, (0xC0 | encode)); 7080 } 7081 7082 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7083 assert(VM_Version::supports_avx(), ""); 7084 InstructionMark im(this); 7085 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7086 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 7087 attributes.set_rex_vex_w_reverted(); 7088 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7089 emit_int8(0x57); 7090 emit_operand(dst, src, 0); 7091 } 7092 7093 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7094 assert(VM_Version::supports_avx(), ""); 7095 InstructionMark im(this); 7096 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7097 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7098 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7099 emit_int8(0x57); 7100 emit_operand(dst, src, 0); 7101 } 7102 7103 // Integer vector arithmetic 7104 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7105 assert(VM_Version::supports_avx() && (vector_len == 0) || 7106 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2"); 7107 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 7108 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7109 emit_int16(0x01, (0xC0 | encode)); 7110 } 7111 7112 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7113 assert(VM_Version::supports_avx() && (vector_len == 0) || 7114 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2"); 7115 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 7116 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7117 emit_int16(0x02, (0xC0 | encode)); 7118 } 7119 7120 void Assembler::paddb(XMMRegister dst, XMMRegister src) { 7121 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7122 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7123 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7124 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 7125 } 7126 7127 void Assembler::paddw(XMMRegister dst, XMMRegister src) { 7128 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7129 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7130 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7131 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 7132 } 7133 7134 void Assembler::paddd(XMMRegister dst, XMMRegister src) { 7135 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7137 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7138 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 7139 } 7140 7141 void Assembler::paddd(XMMRegister dst, Address src) { 7142 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7143 InstructionMark im(this); 7144 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7145 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7146 emit_int8((unsigned char)0xFE); 7147 emit_operand(dst, src, 0); 7148 } 7149 7150 void Assembler::paddq(XMMRegister dst, XMMRegister src) { 7151 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7152 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7153 attributes.set_rex_vex_w_reverted(); 7154 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7155 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 7156 } 7157 7158 void Assembler::phaddw(XMMRegister dst, XMMRegister src) { 7159 assert(VM_Version::supports_sse3(), ""); 7160 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 7161 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7162 emit_int16(0x01, (0xC0 | encode)); 7163 } 7164 7165 void Assembler::phaddd(XMMRegister dst, XMMRegister src) { 7166 assert(VM_Version::supports_sse3(), ""); 7167 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 7168 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7169 emit_int16(0x02, (0xC0 | encode)); 7170 } 7171 7172 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7173 assert(UseAVX > 0, "requires some form of AVX"); 7174 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7175 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7176 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 7177 } 7178 7179 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7180 assert(UseAVX > 0, "requires some form of AVX"); 7181 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7182 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7183 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 7184 } 7185 7186 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7187 assert(UseAVX > 0, "requires some form of AVX"); 7188 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7189 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7190 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 7191 } 7192 7193 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7194 assert(UseAVX > 0, "requires some form of AVX"); 7195 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7196 attributes.set_rex_vex_w_reverted(); 7197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7198 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 7199 } 7200 7201 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7202 assert(UseAVX > 0, "requires some form of AVX"); 7203 InstructionMark im(this); 7204 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7205 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 7206 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7207 emit_int8((unsigned char)0xFC); 7208 emit_operand(dst, src, 0); 7209 } 7210 7211 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7212 assert(UseAVX > 0, "requires some form of AVX"); 7213 InstructionMark im(this); 7214 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7215 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 7216 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7217 emit_int8((unsigned char)0xFD); 7218 emit_operand(dst, src, 0); 7219 } 7220 7221 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7222 assert(UseAVX > 0, "requires some form of AVX"); 7223 InstructionMark im(this); 7224 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7225 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7226 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7227 emit_int8((unsigned char)0xFE); 7228 emit_operand(dst, src, 0); 7229 } 7230 7231 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7232 assert(UseAVX > 0, "requires some form of AVX"); 7233 InstructionMark im(this); 7234 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7235 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 7236 attributes.set_rex_vex_w_reverted(); 7237 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7238 emit_int8((unsigned char)0xD4); 7239 emit_operand(dst, src, 0); 7240 } 7241 7242 void Assembler::psubb(XMMRegister dst, XMMRegister src) { 7243 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7244 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7245 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7246 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 7247 } 7248 7249 void Assembler::psubw(XMMRegister dst, XMMRegister src) { 7250 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7251 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7252 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7253 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 7254 } 7255 7256 void Assembler::psubd(XMMRegister dst, XMMRegister src) { 7257 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7258 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7259 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 7260 } 7261 7262 void Assembler::psubq(XMMRegister dst, XMMRegister src) { 7263 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7264 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7265 attributes.set_rex_vex_w_reverted(); 7266 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7267 emit_int8((unsigned char)0xFB); 7268 emit_int8((0xC0 | encode)); 7269 } 7270 7271 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7272 assert(UseAVX > 0, "requires some form of AVX"); 7273 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7275 emit_int16((unsigned char)0xD8, (0xC0 | encode)); 7276 } 7277 7278 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7279 assert(UseAVX > 0, "requires some form of AVX"); 7280 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7282 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 7283 } 7284 7285 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7286 assert(UseAVX > 0, "requires some form of AVX"); 7287 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7288 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7289 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 7290 } 7291 7292 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7293 assert(UseAVX > 0, "requires some form of AVX"); 7294 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7295 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7296 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 7297 } 7298 7299 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7300 assert(UseAVX > 0, "requires some form of AVX"); 7301 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7302 attributes.set_rex_vex_w_reverted(); 7303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7304 emit_int16((unsigned char)0xFB, (0xC0 | encode)); 7305 } 7306 7307 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7308 assert(UseAVX > 0, "requires some form of AVX"); 7309 InstructionMark im(this); 7310 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7311 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 7312 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7313 emit_int8((unsigned char)0xF8); 7314 emit_operand(dst, src, 0); 7315 } 7316 7317 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7318 assert(UseAVX > 0, "requires some form of AVX"); 7319 InstructionMark im(this); 7320 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7321 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 7322 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7323 emit_int8((unsigned char)0xF9); 7324 emit_operand(dst, src, 0); 7325 } 7326 7327 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7328 assert(UseAVX > 0, "requires some form of AVX"); 7329 InstructionMark im(this); 7330 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7331 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7332 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7333 emit_int8((unsigned char)0xFA); 7334 emit_operand(dst, src, 0); 7335 } 7336 7337 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7338 assert(UseAVX > 0, "requires some form of AVX"); 7339 InstructionMark im(this); 7340 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7341 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 7342 attributes.set_rex_vex_w_reverted(); 7343 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7344 emit_int8((unsigned char)0xFB); 7345 emit_operand(dst, src, 0); 7346 } 7347 7348 void Assembler::pmullw(XMMRegister dst, XMMRegister src) { 7349 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7351 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7352 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 7353 } 7354 7355 void Assembler::pmulld(XMMRegister dst, XMMRegister src) { 7356 assert(VM_Version::supports_sse4_1(), ""); 7357 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7358 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7359 emit_int16(0x40, (0xC0 | encode)); 7360 } 7361 7362 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) { 7363 assert(VM_Version::supports_sse2(), ""); 7364 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7366 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 7367 } 7368 7369 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7370 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) || 7371 (vector_len == AVX_256bit && VM_Version::supports_avx2()) || 7372 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), ""); 7373 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7374 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7375 emit_int16((unsigned char)0xE4, (0xC0 | encode)); 7376 } 7377 7378 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7379 assert(UseAVX > 0, "requires some form of AVX"); 7380 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7381 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7382 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 7383 } 7384 7385 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7386 assert(UseAVX > 0, "requires some form of AVX"); 7387 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7388 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7389 emit_int16(0x40, (0xC0 | encode)); 7390 } 7391 7392 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7393 assert(UseAVX > 2, "requires some form of EVEX"); 7394 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7395 attributes.set_is_evex_instruction(); 7396 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7397 emit_int16(0x40, (0xC0 | encode)); 7398 } 7399 7400 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7401 assert(UseAVX > 0, "requires some form of AVX"); 7402 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7403 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7404 emit_int16((unsigned char)0xF4, (0xC0 | encode)); 7405 } 7406 7407 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7408 assert(UseAVX > 0, "requires some form of AVX"); 7409 InstructionMark im(this); 7410 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7411 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 7412 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7413 emit_int8((unsigned char)0xD5); 7414 emit_operand(dst, src, 0); 7415 } 7416 7417 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7418 assert(UseAVX > 0, "requires some form of AVX"); 7419 InstructionMark im(this); 7420 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7421 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7422 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7423 emit_int8(0x40); 7424 emit_operand(dst, src, 0); 7425 } 7426 7427 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7428 assert(UseAVX > 2, "requires some form of EVEX"); 7429 InstructionMark im(this); 7430 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true); 7431 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 7432 attributes.set_is_evex_instruction(); 7433 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7434 emit_int8(0x40); 7435 emit_operand(dst, src, 0); 7436 } 7437 7438 // Min, max 7439 void Assembler::pminsb(XMMRegister dst, XMMRegister src) { 7440 assert(VM_Version::supports_sse4_1(), ""); 7441 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7442 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7443 emit_int16(0x38, (0xC0 | encode)); 7444 } 7445 7446 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7447 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7448 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 7449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7450 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7451 emit_int16(0x38, (0xC0 | encode)); 7452 } 7453 7454 void Assembler::pminsw(XMMRegister dst, XMMRegister src) { 7455 assert(VM_Version::supports_sse2(), ""); 7456 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7457 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7458 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 7459 } 7460 7461 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7462 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7463 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 7464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7465 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7466 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 7467 } 7468 7469 void Assembler::pminsd(XMMRegister dst, XMMRegister src) { 7470 assert(VM_Version::supports_sse4_1(), ""); 7471 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7472 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7473 emit_int16(0x39, (0xC0 | encode)); 7474 } 7475 7476 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7477 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7478 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), ""); 7479 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7481 emit_int16(0x39, (0xC0 | encode)); 7482 } 7483 7484 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7485 assert(UseAVX > 2, "requires AVX512F"); 7486 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7487 attributes.set_is_evex_instruction(); 7488 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7489 emit_int16(0x39, (0xC0 | encode)); 7490 } 7491 7492 void Assembler::minps(XMMRegister dst, XMMRegister src) { 7493 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7494 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7495 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7496 emit_int16(0x5D, (0xC0 | encode)); 7497 } 7498 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7499 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 7500 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7501 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7502 emit_int16(0x5D, (0xC0 | encode)); 7503 } 7504 7505 void Assembler::minpd(XMMRegister dst, XMMRegister src) { 7506 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7508 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7509 emit_int16(0x5D, (0xC0 | encode)); 7510 } 7511 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7512 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 7513 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7514 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7515 emit_int16(0x5D, (0xC0 | encode)); 7516 } 7517 7518 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) { 7519 assert(VM_Version::supports_sse4_1(), ""); 7520 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7521 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7522 emit_int16(0x3C, (0xC0 | encode)); 7523 } 7524 7525 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7526 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7527 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 7528 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7530 emit_int16(0x3C, (0xC0 | encode)); 7531 } 7532 7533 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) { 7534 assert(VM_Version::supports_sse2(), ""); 7535 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7536 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7537 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 7538 } 7539 7540 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7541 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7542 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), ""); 7543 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7544 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7545 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 7546 } 7547 7548 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) { 7549 assert(VM_Version::supports_sse4_1(), ""); 7550 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7551 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7552 emit_int16(0x3D, (0xC0 | encode)); 7553 } 7554 7555 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7556 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 7557 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), ""); 7558 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7559 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7560 emit_int16(0x3D, (0xC0 | encode)); 7561 } 7562 7563 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7564 assert(UseAVX > 2, "requires AVX512F"); 7565 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7566 attributes.set_is_evex_instruction(); 7567 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7568 emit_int16(0x3D, (0xC0 | encode)); 7569 } 7570 7571 void Assembler::maxps(XMMRegister dst, XMMRegister src) { 7572 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7573 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7574 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7575 emit_int16(0x5F, (0xC0 | encode)); 7576 } 7577 7578 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7579 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 7580 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7581 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 7582 emit_int16(0x5F, (0xC0 | encode)); 7583 } 7584 7585 void Assembler::maxpd(XMMRegister dst, XMMRegister src) { 7586 NOT_LP64(assert(VM_Version::supports_sse(), "")); 7587 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7588 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7589 emit_int16(0x5F, (0xC0 | encode)); 7590 } 7591 7592 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7593 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), ""); 7594 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7596 emit_int16(0x5F, (0xC0 | encode)); 7597 } 7598 7599 // Shift packed integers left by specified number of bits. 7600 void Assembler::psllw(XMMRegister dst, int shift) { 7601 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7602 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7603 // XMM6 is for /6 encoding: 66 0F 71 /6 ib 7604 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7605 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7606 } 7607 7608 void Assembler::pslld(XMMRegister dst, int shift) { 7609 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7611 // XMM6 is for /6 encoding: 66 0F 72 /6 ib 7612 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7613 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 7614 } 7615 7616 void Assembler::psllq(XMMRegister dst, int shift) { 7617 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7618 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7619 // XMM6 is for /6 encoding: 66 0F 73 /6 ib 7620 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7621 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 7622 } 7623 7624 void Assembler::psllw(XMMRegister dst, XMMRegister shift) { 7625 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7626 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7627 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7628 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 7629 } 7630 7631 void Assembler::pslld(XMMRegister dst, XMMRegister shift) { 7632 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7633 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7634 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7635 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 7636 } 7637 7638 void Assembler::psllq(XMMRegister dst, XMMRegister shift) { 7639 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7640 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7641 attributes.set_rex_vex_w_reverted(); 7642 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7643 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 7644 } 7645 7646 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7647 assert(UseAVX > 0, "requires some form of AVX"); 7648 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7649 // XMM6 is for /6 encoding: 66 0F 71 /6 ib 7650 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7651 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7652 } 7653 7654 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7655 assert(UseAVX > 0, "requires some form of AVX"); 7656 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7657 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7658 // XMM6 is for /6 encoding: 66 0F 72 /6 ib 7659 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7660 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 7661 } 7662 7663 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7664 assert(UseAVX > 0, "requires some form of AVX"); 7665 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7666 attributes.set_rex_vex_w_reverted(); 7667 // XMM6 is for /6 encoding: 66 0F 73 /6 ib 7668 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7669 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 7670 } 7671 7672 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7673 assert(UseAVX > 0, "requires some form of AVX"); 7674 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7675 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7676 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 7677 } 7678 7679 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7680 assert(UseAVX > 0, "requires some form of AVX"); 7681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7682 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7683 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 7684 } 7685 7686 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7687 assert(UseAVX > 0, "requires some form of AVX"); 7688 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7689 attributes.set_rex_vex_w_reverted(); 7690 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7691 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 7692 } 7693 7694 // Shift packed integers logically right by specified number of bits. 7695 void Assembler::psrlw(XMMRegister dst, int shift) { 7696 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7697 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7698 // XMM2 is for /2 encoding: 66 0F 71 /2 ib 7699 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7700 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7701 } 7702 7703 void Assembler::psrld(XMMRegister dst, int shift) { 7704 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7705 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7706 // XMM2 is for /2 encoding: 66 0F 72 /2 ib 7707 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7708 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 7709 } 7710 7711 void Assembler::psrlq(XMMRegister dst, int shift) { 7712 // Do not confuse it with psrldq SSE2 instruction which 7713 // shifts 128 bit value in xmm register by number of bytes. 7714 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7715 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7716 attributes.set_rex_vex_w_reverted(); 7717 // XMM2 is for /2 encoding: 66 0F 73 /2 ib 7718 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7719 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 7720 } 7721 7722 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) { 7723 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7725 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7726 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 7727 } 7728 7729 void Assembler::psrld(XMMRegister dst, XMMRegister shift) { 7730 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7731 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7732 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7733 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 7734 } 7735 7736 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) { 7737 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7738 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7739 attributes.set_rex_vex_w_reverted(); 7740 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7741 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 7742 } 7743 7744 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7745 assert(UseAVX > 0, "requires some form of AVX"); 7746 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7747 // XMM2 is for /2 encoding: 66 0F 71 /2 ib 7748 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7749 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7750 } 7751 7752 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7753 assert(UseAVX > 0, "requires some form of AVX"); 7754 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7755 // XMM2 is for /2 encoding: 66 0F 72 /2 ib 7756 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7757 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 7758 } 7759 7760 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7761 assert(UseAVX > 0, "requires some form of AVX"); 7762 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7763 attributes.set_rex_vex_w_reverted(); 7764 // XMM2 is for /2 encoding: 66 0F 73 /2 ib 7765 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7766 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 7767 } 7768 7769 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7770 assert(UseAVX > 0, "requires some form of AVX"); 7771 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7772 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7773 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 7774 } 7775 7776 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7777 assert(UseAVX > 0, "requires some form of AVX"); 7778 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7779 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7780 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 7781 } 7782 7783 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7784 assert(UseAVX > 0, "requires some form of AVX"); 7785 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7786 attributes.set_rex_vex_w_reverted(); 7787 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7788 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 7789 } 7790 7791 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7792 assert(VM_Version::supports_avx512bw(), ""); 7793 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7794 attributes.set_is_evex_instruction(); 7795 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7796 emit_int16(0x10, (0xC0 | encode)); 7797 } 7798 7799 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7800 assert(VM_Version::supports_avx512bw(), ""); 7801 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7802 attributes.set_is_evex_instruction(); 7803 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7804 emit_int16(0x12, (0xC0 | encode)); 7805 } 7806 7807 // Shift packed integers arithmetically right by specified number of bits. 7808 void Assembler::psraw(XMMRegister dst, int shift) { 7809 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7810 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7811 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 7812 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7813 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7814 } 7815 7816 void Assembler::psrad(XMMRegister dst, int shift) { 7817 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7818 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7819 // XMM4 is for /4 encoding: 66 0F 72 /4 ib 7820 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7821 emit_int8(0x72); 7822 emit_int8((0xC0 | encode)); 7823 emit_int8(shift & 0xFF); 7824 } 7825 7826 void Assembler::psraw(XMMRegister dst, XMMRegister shift) { 7827 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7828 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7829 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7830 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 7831 } 7832 7833 void Assembler::psrad(XMMRegister dst, XMMRegister shift) { 7834 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7835 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7836 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7837 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 7838 } 7839 7840 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7841 assert(UseAVX > 0, "requires some form of AVX"); 7842 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7843 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 7844 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7845 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 7846 } 7847 7848 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7849 assert(UseAVX > 0, "requires some form of AVX"); 7850 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7851 // XMM4 is for /4 encoding: 66 0F 71 /4 ib 7852 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7853 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 7854 } 7855 7856 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7857 assert(UseAVX > 0, "requires some form of AVX"); 7858 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 7859 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7860 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 7861 } 7862 7863 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7864 assert(UseAVX > 0, "requires some form of AVX"); 7865 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7866 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7867 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 7868 } 7869 7870 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 7871 assert(UseAVX > 2, "requires AVX512"); 7872 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl"); 7873 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7874 attributes.set_is_evex_instruction(); 7875 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7876 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF); 7877 } 7878 7879 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7880 assert(UseAVX > 2, "requires AVX512"); 7881 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl"); 7882 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7883 attributes.set_is_evex_instruction(); 7884 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7885 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 7886 } 7887 7888 // logical operations packed integers 7889 void Assembler::pand(XMMRegister dst, XMMRegister src) { 7890 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7891 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7892 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7893 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 7894 } 7895 7896 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7897 assert(UseAVX > 0, "requires some form of AVX"); 7898 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7899 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7900 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 7901 } 7902 7903 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7904 assert(UseAVX > 0, "requires some form of AVX"); 7905 InstructionMark im(this); 7906 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7907 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 7908 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7909 emit_int8((unsigned char)0xDB); 7910 emit_operand(dst, src, 0); 7911 } 7912 7913 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7914 evpandq(dst, k0, nds, src, false, vector_len); 7915 } 7916 7917 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 7918 evpandq(dst, k0, nds, src, false, vector_len); 7919 } 7920 7921 //Variable Shift packed integers logically left. 7922 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7923 assert(UseAVX > 1, "requires AVX2"); 7924 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7925 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7926 emit_int16(0x47, (0xC0 | encode)); 7927 } 7928 7929 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7930 assert(UseAVX > 1, "requires AVX2"); 7931 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7932 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7933 emit_int16(0x47, (0xC0 | encode)); 7934 } 7935 7936 //Variable Shift packed integers logically right. 7937 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7938 assert(UseAVX > 1, "requires AVX2"); 7939 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7940 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7941 emit_int16(0x45, (0xC0 | encode)); 7942 } 7943 7944 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7945 assert(UseAVX > 1, "requires AVX2"); 7946 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7947 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7948 emit_int16(0x45, (0xC0 | encode)); 7949 } 7950 7951 //Variable right Shift arithmetic packed integers . 7952 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7953 assert(UseAVX > 1, "requires AVX2"); 7954 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7955 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7956 emit_int16(0x46, (0xC0 | encode)); 7957 } 7958 7959 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 7960 assert(VM_Version::supports_avx512bw(), ""); 7961 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7962 attributes.set_is_evex_instruction(); 7963 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7964 emit_int16(0x11, (0xC0 | encode)); 7965 } 7966 7967 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7968 assert(UseAVX > 2, "requires AVX512"); 7969 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 7970 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7971 attributes.set_is_evex_instruction(); 7972 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7973 emit_int16(0x46, (0xC0 | encode)); 7974 } 7975 7976 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7977 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2"); 7978 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7979 attributes.set_is_evex_instruction(); 7980 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7981 emit_int16(0x71, (0xC0 | encode)); 7982 } 7983 7984 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 7985 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2"); 7986 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7987 attributes.set_is_evex_instruction(); 7988 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 7989 emit_int16(0x73, (0xC0 | encode)); 7990 } 7991 7992 void Assembler::pandn(XMMRegister dst, XMMRegister src) { 7993 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 7994 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 7995 attributes.set_rex_vex_w_reverted(); 7996 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 7997 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 7998 } 7999 8000 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8001 assert(UseAVX > 0, "requires some form of AVX"); 8002 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8003 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8004 emit_int16((unsigned char)0xDF, (0xC0 | encode)); 8005 } 8006 8007 void Assembler::por(XMMRegister dst, XMMRegister src) { 8008 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 8009 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8010 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8011 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 8012 } 8013 8014 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8015 assert(UseAVX > 0, "requires some form of AVX"); 8016 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8017 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8018 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 8019 } 8020 8021 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8022 assert(UseAVX > 0, "requires some form of AVX"); 8023 InstructionMark im(this); 8024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8025 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 8026 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8027 emit_int8((unsigned char)0xEB); 8028 emit_operand(dst, src, 0); 8029 } 8030 8031 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8032 evporq(dst, k0, nds, src, false, vector_len); 8033 } 8034 8035 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8036 evporq(dst, k0, nds, src, false, vector_len); 8037 } 8038 8039 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8040 assert(VM_Version::supports_evex(), ""); 8041 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r 8042 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8043 attributes.set_is_evex_instruction(); 8044 attributes.set_embedded_opmask_register_specifier(mask); 8045 if (merge) { 8046 attributes.reset_is_clear_context(); 8047 } 8048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8049 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 8050 } 8051 8052 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8053 assert(VM_Version::supports_evex(), ""); 8054 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r 8055 InstructionMark im(this); 8056 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8057 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 8058 attributes.set_is_evex_instruction(); 8059 attributes.set_embedded_opmask_register_specifier(mask); 8060 if (merge) { 8061 attributes.reset_is_clear_context(); 8062 } 8063 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8064 emit_int8((unsigned char)0xEB); 8065 emit_operand(dst, src, 0); 8066 } 8067 8068 void Assembler::pxor(XMMRegister dst, XMMRegister src) { 8069 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 8070 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8071 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8072 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8073 } 8074 8075 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8076 assert(UseAVX > 0, "requires some form of AVX"); 8077 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8078 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 8079 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, ""); 8080 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8081 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8082 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8083 } 8084 8085 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8086 assert(UseAVX > 0, "requires some form of AVX"); 8087 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : 8088 vector_len == AVX_256bit ? VM_Version::supports_avx2() : 8089 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, ""); 8090 InstructionMark im(this); 8091 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8092 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit); 8093 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8094 emit_int8((unsigned char)0xEF); 8095 emit_operand(dst, src, 0); 8096 } 8097 8098 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8099 assert(UseAVX > 2, "requires some form of EVEX"); 8100 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8101 attributes.set_rex_vex_w_reverted(); 8102 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8103 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8104 } 8105 8106 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8107 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r 8108 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8109 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8110 attributes.set_is_evex_instruction(); 8111 attributes.set_embedded_opmask_register_specifier(mask); 8112 if (merge) { 8113 attributes.reset_is_clear_context(); 8114 } 8115 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8116 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8117 } 8118 8119 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8120 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8121 InstructionMark im(this); 8122 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8123 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8124 attributes.set_is_evex_instruction(); 8125 attributes.set_embedded_opmask_register_specifier(mask); 8126 if (merge) { 8127 attributes.reset_is_clear_context(); 8128 } 8129 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8130 emit_int8((unsigned char)0xEF); 8131 emit_operand(dst, src, 0); 8132 } 8133 8134 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8135 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r 8136 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8137 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8138 attributes.set_is_evex_instruction(); 8139 attributes.set_embedded_opmask_register_specifier(mask); 8140 if (merge) { 8141 attributes.reset_is_clear_context(); 8142 } 8143 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8144 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8145 } 8146 8147 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8148 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8149 InstructionMark im(this); 8150 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8151 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8152 attributes.set_is_evex_instruction(); 8153 attributes.set_embedded_opmask_register_specifier(mask); 8154 if (merge) { 8155 attributes.reset_is_clear_context(); 8156 } 8157 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8158 emit_int8((unsigned char)0xEF); 8159 emit_operand(dst, src, 0); 8160 } 8161 8162 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8163 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8164 InstructionMark im(this); 8165 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8166 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8167 attributes.set_is_evex_instruction(); 8168 attributes.set_embedded_opmask_register_specifier(mask); 8169 if (merge) { 8170 attributes.reset_is_clear_context(); 8171 } 8172 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8173 emit_int8((unsigned char)0xDB); 8174 emit_operand(dst, src, 0); 8175 } 8176 8177 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8178 assert(VM_Version::supports_evex(), "requires AVX512F"); 8179 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 8180 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8181 attributes.set_is_evex_instruction(); 8182 attributes.set_embedded_opmask_register_specifier(mask); 8183 if (merge) { 8184 attributes.reset_is_clear_context(); 8185 } 8186 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8187 emit_int16((unsigned char)0xDB, (0xC0 | encode)); 8188 } 8189 8190 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8191 assert(VM_Version::supports_evex(), "requires AVX512F"); 8192 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 8193 InstructionMark im(this); 8194 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8195 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8196 attributes.set_is_evex_instruction(); 8197 attributes.set_embedded_opmask_register_specifier(mask); 8198 if (merge) { 8199 attributes.reset_is_clear_context(); 8200 } 8201 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8202 emit_int8((unsigned char)0xDB); 8203 emit_operand(dst, src, 0); 8204 } 8205 8206 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8207 assert(VM_Version::supports_evex(), "requires AVX512F"); 8208 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 8209 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 8210 attributes.set_is_evex_instruction(); 8211 attributes.set_embedded_opmask_register_specifier(mask); 8212 if (merge) { 8213 attributes.reset_is_clear_context(); 8214 } 8215 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8216 emit_int16((unsigned char)0xEB, (0xC0 | encode)); 8217 } 8218 8219 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8220 assert(VM_Version::supports_evex(), "requires AVX512F"); 8221 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 8222 InstructionMark im(this); 8223 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8224 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8225 attributes.set_is_evex_instruction(); 8226 attributes.set_embedded_opmask_register_specifier(mask); 8227 if (merge) { 8228 attributes.reset_is_clear_context(); 8229 } 8230 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8231 emit_int8((unsigned char)0xEB); 8232 emit_operand(dst, src, 0); 8233 } 8234 8235 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8236 assert(VM_Version::supports_evex(), "requires EVEX support"); 8237 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8238 attributes.set_is_evex_instruction(); 8239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8240 emit_int16((unsigned char)0xEF, (0xC0 | encode)); 8241 } 8242 8243 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 8244 assert(VM_Version::supports_evex(), "requires EVEX support"); 8245 assert(dst != xnoreg, "sanity"); 8246 InstructionMark im(this); 8247 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8248 attributes.set_is_evex_instruction(); 8249 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 8250 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8251 emit_int8((unsigned char)0xEF); 8252 emit_operand(dst, src, 0); 8253 } 8254 8255 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 8256 assert(VM_Version::supports_evex(), "requires EVEX support"); 8257 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8258 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8259 attributes.set_is_evex_instruction(); 8260 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8261 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 8262 } 8263 8264 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 8265 assert(VM_Version::supports_evex(), "requires EVEX support"); 8266 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8267 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8268 attributes.set_is_evex_instruction(); 8269 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8270 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 8271 } 8272 8273 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 8274 assert(VM_Version::supports_evex(), "requires EVEX support"); 8275 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8276 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8277 attributes.set_is_evex_instruction(); 8278 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8279 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 8280 } 8281 8282 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) { 8283 assert(VM_Version::supports_evex(), "requires EVEX support"); 8284 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8285 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8286 attributes.set_is_evex_instruction(); 8287 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8288 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 8289 } 8290 8291 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 8292 assert(VM_Version::supports_evex(), "requires EVEX support"); 8293 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8294 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8295 attributes.set_is_evex_instruction(); 8296 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8297 emit_int16(0x15, (unsigned char)(0xC0 | encode)); 8298 } 8299 8300 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 8301 assert(VM_Version::supports_evex(), "requires EVEX support"); 8302 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8303 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8304 attributes.set_is_evex_instruction(); 8305 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8306 emit_int16(0x15, (unsigned char)(0xC0 | encode)); 8307 } 8308 8309 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 8310 assert(VM_Version::supports_evex(), "requires EVEX support"); 8311 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8312 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8313 attributes.set_is_evex_instruction(); 8314 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8315 emit_int16(0x14, (unsigned char)(0xC0 | encode)); 8316 } 8317 8318 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) { 8319 assert(VM_Version::supports_evex(), "requires EVEX support"); 8320 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8321 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8322 attributes.set_is_evex_instruction(); 8323 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8324 emit_int16(0x14, (unsigned char)(0xC0 | encode)); 8325 } 8326 8327 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8328 assert(VM_Version::supports_avx512cd(), ""); 8329 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8330 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8331 attributes.set_is_evex_instruction(); 8332 attributes.set_embedded_opmask_register_specifier(mask); 8333 if (merge) { 8334 attributes.reset_is_clear_context(); 8335 } 8336 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8337 emit_int16(0x44, (0xC0 | encode)); 8338 } 8339 8340 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8341 assert(VM_Version::supports_avx512cd(), ""); 8342 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8343 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8344 attributes.set_is_evex_instruction(); 8345 attributes.set_embedded_opmask_register_specifier(mask); 8346 if (merge) { 8347 attributes.reset_is_clear_context(); 8348 } 8349 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8350 emit_int16(0x44, (0xC0 | encode)); 8351 } 8352 8353 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) { 8354 assert(VM_Version::supports_evex(), "requires EVEX support"); 8355 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8356 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8357 attributes.set_is_evex_instruction(); 8358 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8359 emit_int8(0x25); 8360 emit_int8((unsigned char)(0xC0 | encode)); 8361 emit_int8(imm8); 8362 } 8363 8364 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) { 8365 assert(VM_Version::supports_evex(), "requires EVEX support"); 8366 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8367 assert(dst != xnoreg, "sanity"); 8368 InstructionMark im(this); 8369 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8370 attributes.set_is_evex_instruction(); 8371 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 8372 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8373 emit_int8(0x25); 8374 emit_operand(dst, src3, 1); 8375 emit_int8(imm8); 8376 } 8377 8378 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) { 8379 assert(VM_Version::supports_evex(), "requires AVX512F"); 8380 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL"); 8381 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8382 attributes.set_is_evex_instruction(); 8383 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8384 emit_int8(0x25); 8385 emit_int8((unsigned char)(0xC0 | encode)); 8386 emit_int8(imm8); 8387 } 8388 8389 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) { 8390 assert(VM_Version::supports_evex(), "requires EVEX support"); 8391 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 8392 assert(dst != xnoreg, "sanity"); 8393 InstructionMark im(this); 8394 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8395 attributes.set_is_evex_instruction(); 8396 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 8397 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8398 emit_int8(0x25); 8399 emit_operand(dst, src3, 1); 8400 emit_int8(imm8); 8401 } 8402 8403 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8404 assert(VM_Version::supports_evex(), ""); 8405 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8406 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8407 attributes.set_is_evex_instruction(); 8408 attributes.set_embedded_opmask_register_specifier(mask); 8409 if (merge) { 8410 attributes.reset_is_clear_context(); 8411 } 8412 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8413 emit_int16((unsigned char)0x88, (0xC0 | encode)); 8414 } 8415 8416 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8417 assert(VM_Version::supports_evex(), ""); 8418 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8419 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8420 attributes.set_is_evex_instruction(); 8421 attributes.set_embedded_opmask_register_specifier(mask); 8422 if (merge) { 8423 attributes.reset_is_clear_context(); 8424 } 8425 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8426 emit_int16((unsigned char)0x88, (0xC0 | encode)); 8427 } 8428 8429 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8430 assert(VM_Version::supports_avx512_vbmi2(), ""); 8431 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8432 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8433 attributes.set_is_evex_instruction(); 8434 attributes.set_embedded_opmask_register_specifier(mask); 8435 if (merge) { 8436 attributes.reset_is_clear_context(); 8437 } 8438 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8439 emit_int16(0x62, (0xC0 | encode)); 8440 } 8441 8442 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8443 assert(VM_Version::supports_avx512_vbmi2(), ""); 8444 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8445 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8446 attributes.set_is_evex_instruction(); 8447 attributes.set_embedded_opmask_register_specifier(mask); 8448 if (merge) { 8449 attributes.reset_is_clear_context(); 8450 } 8451 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8452 emit_int16(0x62, (0xC0 | encode)); 8453 } 8454 8455 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8456 assert(VM_Version::supports_evex(), ""); 8457 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8458 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8459 attributes.set_is_evex_instruction(); 8460 attributes.set_embedded_opmask_register_specifier(mask); 8461 if (merge) { 8462 attributes.reset_is_clear_context(); 8463 } 8464 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8465 emit_int16((unsigned char)0x89, (0xC0 | encode)); 8466 } 8467 8468 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 8469 assert(VM_Version::supports_evex(), ""); 8470 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8471 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8472 attributes.set_is_evex_instruction(); 8473 attributes.set_embedded_opmask_register_specifier(mask); 8474 if (merge) { 8475 attributes.reset_is_clear_context(); 8476 } 8477 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8478 emit_int16((unsigned char)0x89, (0xC0 | encode)); 8479 } 8480 8481 // vinserti forms 8482 8483 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8484 assert(VM_Version::supports_avx2(), ""); 8485 assert(imm8 <= 0x01, "imm8: %u", imm8); 8486 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8487 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8488 // last byte: 8489 // 0x00 - insert into lower 128 bits 8490 // 0x01 - insert into upper 128 bits 8491 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01); 8492 } 8493 8494 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 8495 assert(VM_Version::supports_avx2(), ""); 8496 assert(dst != xnoreg, "sanity"); 8497 assert(imm8 <= 0x01, "imm8: %u", imm8); 8498 InstructionMark im(this); 8499 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8500 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8501 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8502 emit_int8(0x38); 8503 emit_operand(dst, src, 1); 8504 // 0x00 - insert into lower 128 bits 8505 // 0x01 - insert into upper 128 bits 8506 emit_int8(imm8 & 0x01); 8507 } 8508 8509 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8510 assert(VM_Version::supports_evex(), ""); 8511 assert(imm8 <= 0x03, "imm8: %u", imm8); 8512 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8513 attributes.set_is_evex_instruction(); 8514 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8515 // imm8: 8516 // 0x00 - insert into q0 128 bits (0..127) 8517 // 0x01 - insert into q1 128 bits (128..255) 8518 // 0x02 - insert into q2 128 bits (256..383) 8519 // 0x03 - insert into q3 128 bits (384..511) 8520 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03); 8521 } 8522 8523 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 8524 assert(VM_Version::supports_evex(), ""); 8525 assert(dst != xnoreg, "sanity"); 8526 assert(imm8 <= 0x03, "imm8: %u", imm8); 8527 InstructionMark im(this); 8528 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8529 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8530 attributes.set_is_evex_instruction(); 8531 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8532 emit_int8(0x18); 8533 emit_operand(dst, src, 1); 8534 // 0x00 - insert into q0 128 bits (0..127) 8535 // 0x01 - insert into q1 128 bits (128..255) 8536 // 0x02 - insert into q2 128 bits (256..383) 8537 // 0x03 - insert into q3 128 bits (384..511) 8538 emit_int8(imm8 & 0x03); 8539 } 8540 8541 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8542 assert(VM_Version::supports_evex(), ""); 8543 assert(imm8 <= 0x01, "imm8: %u", imm8); 8544 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8545 attributes.set_is_evex_instruction(); 8546 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8547 //imm8: 8548 // 0x00 - insert into lower 256 bits 8549 // 0x01 - insert into upper 256 bits 8550 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01); 8551 } 8552 8553 8554 // vinsertf forms 8555 8556 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8557 assert(VM_Version::supports_avx(), ""); 8558 assert(imm8 <= 0x01, "imm8: %u", imm8); 8559 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8560 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8561 // imm8: 8562 // 0x00 - insert into lower 128 bits 8563 // 0x01 - insert into upper 128 bits 8564 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01); 8565 } 8566 8567 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 8568 assert(VM_Version::supports_avx(), ""); 8569 assert(dst != xnoreg, "sanity"); 8570 assert(imm8 <= 0x01, "imm8: %u", imm8); 8571 InstructionMark im(this); 8572 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8573 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8574 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8575 emit_int8(0x18); 8576 emit_operand(dst, src, 1); 8577 // 0x00 - insert into lower 128 bits 8578 // 0x01 - insert into upper 128 bits 8579 emit_int8(imm8 & 0x01); 8580 } 8581 8582 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8583 assert(VM_Version::supports_evex(), ""); 8584 assert(imm8 <= 0x03, "imm8: %u", imm8); 8585 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8586 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8587 // imm8: 8588 // 0x00 - insert into q0 128 bits (0..127) 8589 // 0x01 - insert into q1 128 bits (128..255) 8590 // 0x02 - insert into q0 128 bits (256..383) 8591 // 0x03 - insert into q1 128 bits (384..512) 8592 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03); 8593 } 8594 8595 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 8596 assert(VM_Version::supports_evex(), ""); 8597 assert(dst != xnoreg, "sanity"); 8598 assert(imm8 <= 0x03, "imm8: %u", imm8); 8599 InstructionMark im(this); 8600 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8601 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8602 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8603 emit_int8(0x18); 8604 emit_operand(dst, src, 1); 8605 // 0x00 - insert into q0 128 bits (0..127) 8606 // 0x01 - insert into q1 128 bits (128..255) 8607 // 0x02 - insert into q0 128 bits (256..383) 8608 // 0x03 - insert into q1 128 bits (384..512) 8609 emit_int8(imm8 & 0x03); 8610 } 8611 8612 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) { 8613 assert(VM_Version::supports_evex(), ""); 8614 assert(imm8 <= 0x01, "imm8: %u", imm8); 8615 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8616 attributes.set_is_evex_instruction(); 8617 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8618 // imm8: 8619 // 0x00 - insert into lower 256 bits 8620 // 0x01 - insert into upper 256 bits 8621 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01); 8622 } 8623 8624 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) { 8625 assert(VM_Version::supports_evex(), ""); 8626 assert(dst != xnoreg, "sanity"); 8627 assert(imm8 <= 0x01, "imm8: %u", imm8); 8628 InstructionMark im(this); 8629 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8630 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit); 8631 attributes.set_is_evex_instruction(); 8632 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8633 emit_int8(0x1A); 8634 emit_operand(dst, src, 1); 8635 // 0x00 - insert into lower 256 bits 8636 // 0x01 - insert into upper 256 bits 8637 emit_int8(imm8 & 0x01); 8638 } 8639 8640 8641 // vextracti forms 8642 8643 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8644 assert(VM_Version::supports_avx2(), ""); 8645 assert(imm8 <= 0x01, "imm8: %u", imm8); 8646 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8647 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8648 // imm8: 8649 // 0x00 - extract from lower 128 bits 8650 // 0x01 - extract from upper 128 bits 8651 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01); 8652 } 8653 8654 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) { 8655 assert(VM_Version::supports_avx2(), ""); 8656 assert(src != xnoreg, "sanity"); 8657 assert(imm8 <= 0x01, "imm8: %u", imm8); 8658 InstructionMark im(this); 8659 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8660 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8661 attributes.reset_is_clear_context(); 8662 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8663 emit_int8(0x39); 8664 emit_operand(src, dst, 1); 8665 // 0x00 - extract from lower 128 bits 8666 // 0x01 - extract from upper 128 bits 8667 emit_int8(imm8 & 0x01); 8668 } 8669 8670 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8671 assert(VM_Version::supports_evex(), ""); 8672 assert(imm8 <= 0x03, "imm8: %u", imm8); 8673 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8674 attributes.set_is_evex_instruction(); 8675 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8676 // imm8: 8677 // 0x00 - extract from bits 127:0 8678 // 0x01 - extract from bits 255:128 8679 // 0x02 - extract from bits 383:256 8680 // 0x03 - extract from bits 511:384 8681 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03); 8682 } 8683 8684 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) { 8685 assert(VM_Version::supports_evex(), ""); 8686 assert(src != xnoreg, "sanity"); 8687 assert(imm8 <= 0x03, "imm8: %u", imm8); 8688 InstructionMark im(this); 8689 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8690 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8691 attributes.reset_is_clear_context(); 8692 attributes.set_is_evex_instruction(); 8693 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8694 emit_int8(0x39); 8695 emit_operand(src, dst, 1); 8696 // 0x00 - extract from bits 127:0 8697 // 0x01 - extract from bits 255:128 8698 // 0x02 - extract from bits 383:256 8699 // 0x03 - extract from bits 511:384 8700 emit_int8(imm8 & 0x03); 8701 } 8702 8703 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8704 assert(VM_Version::supports_avx512dq(), ""); 8705 assert(imm8 <= 0x03, "imm8: %u", imm8); 8706 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8707 attributes.set_is_evex_instruction(); 8708 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8709 // imm8: 8710 // 0x00 - extract from bits 127:0 8711 // 0x01 - extract from bits 255:128 8712 // 0x02 - extract from bits 383:256 8713 // 0x03 - extract from bits 511:384 8714 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03); 8715 } 8716 8717 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8718 assert(VM_Version::supports_evex(), ""); 8719 assert(imm8 <= 0x01, "imm8: %u", imm8); 8720 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8721 attributes.set_is_evex_instruction(); 8722 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8723 // imm8: 8724 // 0x00 - extract from lower 256 bits 8725 // 0x01 - extract from upper 256 bits 8726 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01); 8727 } 8728 8729 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) { 8730 assert(VM_Version::supports_evex(), ""); 8731 assert(src != xnoreg, "sanity"); 8732 assert(imm8 <= 0x01, "imm8: %u", imm8); 8733 InstructionMark im(this); 8734 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8735 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit); 8736 attributes.reset_is_clear_context(); 8737 attributes.set_is_evex_instruction(); 8738 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8739 emit_int8(0x38); 8740 emit_operand(src, dst, 1); 8741 // 0x00 - extract from lower 256 bits 8742 // 0x01 - extract from upper 256 bits 8743 emit_int8(imm8 & 0x01); 8744 } 8745 // vextractf forms 8746 8747 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8748 assert(VM_Version::supports_avx(), ""); 8749 assert(imm8 <= 0x01, "imm8: %u", imm8); 8750 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8751 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8752 // imm8: 8753 // 0x00 - extract from lower 128 bits 8754 // 0x01 - extract from upper 128 bits 8755 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01); 8756 } 8757 8758 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) { 8759 assert(VM_Version::supports_avx(), ""); 8760 assert(src != xnoreg, "sanity"); 8761 assert(imm8 <= 0x01, "imm8: %u", imm8); 8762 InstructionMark im(this); 8763 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8764 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8765 attributes.reset_is_clear_context(); 8766 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8767 emit_int8(0x19); 8768 emit_operand(src, dst, 1); 8769 // 0x00 - extract from lower 128 bits 8770 // 0x01 - extract from upper 128 bits 8771 emit_int8(imm8 & 0x01); 8772 } 8773 8774 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8775 assert(VM_Version::supports_evex(), ""); 8776 assert(imm8 <= 0x03, "imm8: %u", imm8); 8777 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8778 attributes.set_is_evex_instruction(); 8779 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8780 // imm8: 8781 // 0x00 - extract from bits 127:0 8782 // 0x01 - extract from bits 255:128 8783 // 0x02 - extract from bits 383:256 8784 // 0x03 - extract from bits 511:384 8785 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03); 8786 } 8787 8788 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) { 8789 assert(VM_Version::supports_evex(), ""); 8790 assert(src != xnoreg, "sanity"); 8791 assert(imm8 <= 0x03, "imm8: %u", imm8); 8792 InstructionMark im(this); 8793 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8794 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 8795 attributes.reset_is_clear_context(); 8796 attributes.set_is_evex_instruction(); 8797 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8798 emit_int8(0x19); 8799 emit_operand(src, dst, 1); 8800 // 0x00 - extract from bits 127:0 8801 // 0x01 - extract from bits 255:128 8802 // 0x02 - extract from bits 383:256 8803 // 0x03 - extract from bits 511:384 8804 emit_int8(imm8 & 0x03); 8805 } 8806 8807 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8808 assert(VM_Version::supports_avx512dq(), ""); 8809 assert(imm8 <= 0x03, "imm8: %u", imm8); 8810 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8811 attributes.set_is_evex_instruction(); 8812 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8813 // imm8: 8814 // 0x00 - extract from bits 127:0 8815 // 0x01 - extract from bits 255:128 8816 // 0x02 - extract from bits 383:256 8817 // 0x03 - extract from bits 511:384 8818 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03); 8819 } 8820 8821 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) { 8822 assert(VM_Version::supports_evex(), ""); 8823 assert(imm8 <= 0x01, "imm8: %u", imm8); 8824 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8825 attributes.set_is_evex_instruction(); 8826 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8827 // imm8: 8828 // 0x00 - extract from lower 256 bits 8829 // 0x01 - extract from upper 256 bits 8830 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01); 8831 } 8832 8833 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) { 8834 assert(VM_Version::supports_evex(), ""); 8835 assert(src != xnoreg, "sanity"); 8836 assert(imm8 <= 0x01, "imm8: %u", imm8); 8837 InstructionMark im(this); 8838 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8839 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit); 8840 attributes.reset_is_clear_context(); 8841 attributes.set_is_evex_instruction(); 8842 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 8843 emit_int8(0x1B); 8844 emit_operand(src, dst, 1); 8845 // 0x00 - extract from lower 256 bits 8846 // 0x01 - extract from upper 256 bits 8847 emit_int8(imm8 & 0x01); 8848 } 8849 8850 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 8851 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) { 8852 assert(VM_Version::supports_avx2(), ""); 8853 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8854 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8855 emit_int16(0x78, (0xC0 | encode)); 8856 } 8857 8858 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) { 8859 assert(VM_Version::supports_avx2(), ""); 8860 assert(dst != xnoreg, "sanity"); 8861 InstructionMark im(this); 8862 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8863 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit); 8864 // swap src<->dst for encoding 8865 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8866 emit_int8(0x78); 8867 emit_operand(dst, src, 0); 8868 } 8869 8870 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 8871 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 8872 assert(VM_Version::supports_avx2(), ""); 8873 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8874 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8875 emit_int16(0x79, (0xC0 | encode)); 8876 } 8877 8878 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) { 8879 assert(VM_Version::supports_avx2(), ""); 8880 assert(dst != xnoreg, "sanity"); 8881 InstructionMark im(this); 8882 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8883 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit); 8884 // swap src<->dst for encoding 8885 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 8886 emit_int8(0x79); 8887 emit_operand(dst, src, 0); 8888 } 8889 8890 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8891 assert(UseAVX > 0, "requires some form of AVX"); 8892 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 8893 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8894 emit_int16((unsigned char)0xF6, (0xC0 | encode)); 8895 } 8896 8897 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8898 assert(UseAVX > 0, "requires some form of AVX"); 8899 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8900 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8901 emit_int16(0x69, (0xC0 | encode)); 8902 } 8903 8904 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8905 assert(UseAVX > 0, "requires some form of AVX"); 8906 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8907 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8908 emit_int16(0x61, (0xC0 | encode)); 8909 } 8910 8911 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8912 assert(UseAVX > 0, "requires some form of AVX"); 8913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8914 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8915 emit_int16(0x6A, (0xC0 | encode)); 8916 } 8917 8918 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 8919 assert(UseAVX > 0, "requires some form of AVX"); 8920 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 8921 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8922 emit_int16(0x62, (0xC0 | encode)); 8923 } 8924 8925 // xmm/mem sourced byte/word/dword/qword replicate 8926 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8927 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8928 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8929 attributes.set_is_evex_instruction(); 8930 attributes.set_embedded_opmask_register_specifier(mask); 8931 if (merge) { 8932 attributes.reset_is_clear_context(); 8933 } 8934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8935 emit_int16((unsigned char)0xFC, (0xC0 | encode)); 8936 } 8937 8938 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8939 InstructionMark im(this); 8940 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8941 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8943 attributes.set_is_evex_instruction(); 8944 attributes.set_embedded_opmask_register_specifier(mask); 8945 if (merge) { 8946 attributes.reset_is_clear_context(); 8947 } 8948 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8949 emit_int8((unsigned char)0xFC); 8950 emit_operand(dst, src, 0); 8951 } 8952 8953 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8954 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8955 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8956 attributes.set_is_evex_instruction(); 8957 attributes.set_embedded_opmask_register_specifier(mask); 8958 if (merge) { 8959 attributes.reset_is_clear_context(); 8960 } 8961 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8962 emit_int16((unsigned char)0xFD, (0xC0 | encode)); 8963 } 8964 8965 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8966 InstructionMark im(this); 8967 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 8968 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8969 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8970 attributes.set_is_evex_instruction(); 8971 attributes.set_embedded_opmask_register_specifier(mask); 8972 if (merge) { 8973 attributes.reset_is_clear_context(); 8974 } 8975 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8976 emit_int8((unsigned char)0xFD); 8977 emit_operand(dst, src, 0); 8978 } 8979 8980 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8981 assert(VM_Version::supports_evex(), ""); 8982 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8983 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8984 attributes.set_is_evex_instruction(); 8985 attributes.set_embedded_opmask_register_specifier(mask); 8986 if (merge) { 8987 attributes.reset_is_clear_context(); 8988 } 8989 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 8990 emit_int16((unsigned char)0xFE, (0xC0 | encode)); 8991 } 8992 8993 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8994 InstructionMark im(this); 8995 assert(VM_Version::supports_evex(), ""); 8996 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 8997 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 8998 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 8999 attributes.set_is_evex_instruction(); 9000 attributes.set_embedded_opmask_register_specifier(mask); 9001 if (merge) { 9002 attributes.reset_is_clear_context(); 9003 } 9004 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9005 emit_int8((unsigned char)0xFE); 9006 emit_operand(dst, src, 0); 9007 } 9008 9009 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9010 assert(VM_Version::supports_evex(), ""); 9011 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9012 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9013 attributes.set_is_evex_instruction(); 9014 attributes.set_embedded_opmask_register_specifier(mask); 9015 if (merge) { 9016 attributes.reset_is_clear_context(); 9017 } 9018 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9019 emit_int16((unsigned char)0xD4, (0xC0 | encode)); 9020 } 9021 9022 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9023 InstructionMark im(this); 9024 assert(VM_Version::supports_evex(), ""); 9025 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9026 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9027 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9028 attributes.set_is_evex_instruction(); 9029 attributes.set_embedded_opmask_register_specifier(mask); 9030 if (merge) { 9031 attributes.reset_is_clear_context(); 9032 } 9033 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9034 emit_int8((unsigned char)0xD4); 9035 emit_operand(dst, src, 0); 9036 } 9037 9038 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9039 assert(VM_Version::supports_evex(), ""); 9040 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9041 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9042 attributes.set_is_evex_instruction(); 9043 attributes.set_embedded_opmask_register_specifier(mask); 9044 if (merge) { 9045 attributes.reset_is_clear_context(); 9046 } 9047 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9048 emit_int16(0x58, (0xC0 | encode)); 9049 } 9050 9051 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9052 InstructionMark im(this); 9053 assert(VM_Version::supports_evex(), ""); 9054 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9055 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9056 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9057 attributes.set_is_evex_instruction(); 9058 attributes.set_embedded_opmask_register_specifier(mask); 9059 if (merge) { 9060 attributes.reset_is_clear_context(); 9061 } 9062 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9063 emit_int8(0x58); 9064 emit_operand(dst, src, 0); 9065 } 9066 9067 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9068 assert(VM_Version::supports_evex(), ""); 9069 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9070 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9071 attributes.set_is_evex_instruction(); 9072 attributes.set_embedded_opmask_register_specifier(mask); 9073 if (merge) { 9074 attributes.reset_is_clear_context(); 9075 } 9076 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9077 emit_int16(0x58, (0xC0 | encode)); 9078 } 9079 9080 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9081 InstructionMark im(this); 9082 assert(VM_Version::supports_evex(), ""); 9083 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9084 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9085 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9086 attributes.set_is_evex_instruction(); 9087 attributes.set_embedded_opmask_register_specifier(mask); 9088 if (merge) { 9089 attributes.reset_is_clear_context(); 9090 } 9091 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9092 emit_int8(0x58); 9093 emit_operand(dst, src, 0); 9094 } 9095 9096 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9097 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9098 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9099 attributes.set_is_evex_instruction(); 9100 attributes.set_embedded_opmask_register_specifier(mask); 9101 if (merge) { 9102 attributes.reset_is_clear_context(); 9103 } 9104 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9105 emit_int16((unsigned char)0xF8, (0xC0 | encode)); 9106 } 9107 9108 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9109 InstructionMark im(this); 9110 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9111 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9112 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9113 attributes.set_is_evex_instruction(); 9114 attributes.set_embedded_opmask_register_specifier(mask); 9115 if (merge) { 9116 attributes.reset_is_clear_context(); 9117 } 9118 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9119 emit_int8((unsigned char)0xF8); 9120 emit_operand(dst, src, 0); 9121 } 9122 9123 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9124 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9126 attributes.set_is_evex_instruction(); 9127 attributes.set_embedded_opmask_register_specifier(mask); 9128 if (merge) { 9129 attributes.reset_is_clear_context(); 9130 } 9131 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9132 emit_int16((unsigned char)0xF9, (0xC0 | encode)); 9133 } 9134 9135 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9136 InstructionMark im(this); 9137 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9138 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9139 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9140 attributes.set_is_evex_instruction(); 9141 attributes.set_embedded_opmask_register_specifier(mask); 9142 if (merge) { 9143 attributes.reset_is_clear_context(); 9144 } 9145 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9146 emit_int8((unsigned char)0xF9); 9147 emit_operand(dst, src, 0); 9148 } 9149 9150 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9151 assert(VM_Version::supports_evex(), ""); 9152 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9153 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9154 attributes.set_is_evex_instruction(); 9155 attributes.set_embedded_opmask_register_specifier(mask); 9156 if (merge) { 9157 attributes.reset_is_clear_context(); 9158 } 9159 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9160 emit_int16((unsigned char)0xFA, (0xC0 | encode)); 9161 } 9162 9163 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9164 InstructionMark im(this); 9165 assert(VM_Version::supports_evex(), ""); 9166 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9167 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9168 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9169 attributes.set_is_evex_instruction(); 9170 attributes.set_embedded_opmask_register_specifier(mask); 9171 if (merge) { 9172 attributes.reset_is_clear_context(); 9173 } 9174 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9175 emit_int8((unsigned char)0xFA); 9176 emit_operand(dst, src, 0); 9177 } 9178 9179 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9180 assert(VM_Version::supports_evex(), ""); 9181 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9182 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9183 attributes.set_is_evex_instruction(); 9184 attributes.set_embedded_opmask_register_specifier(mask); 9185 if (merge) { 9186 attributes.reset_is_clear_context(); 9187 } 9188 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9189 emit_int16((unsigned char)0xFB, (0xC0 | encode)); 9190 } 9191 9192 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9193 InstructionMark im(this); 9194 assert(VM_Version::supports_evex(), ""); 9195 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9196 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9197 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9198 attributes.set_is_evex_instruction(); 9199 attributes.set_embedded_opmask_register_specifier(mask); 9200 if (merge) { 9201 attributes.reset_is_clear_context(); 9202 } 9203 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9204 emit_int8((unsigned char)0xFB); 9205 emit_operand(dst, src, 0); 9206 } 9207 9208 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9209 assert(VM_Version::supports_evex(), ""); 9210 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9211 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9212 attributes.set_is_evex_instruction(); 9213 attributes.set_embedded_opmask_register_specifier(mask); 9214 if (merge) { 9215 attributes.reset_is_clear_context(); 9216 } 9217 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9218 emit_int16(0x5C, (0xC0 | encode)); 9219 } 9220 9221 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9222 InstructionMark im(this); 9223 assert(VM_Version::supports_evex(), ""); 9224 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9225 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9226 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9227 attributes.set_is_evex_instruction(); 9228 attributes.set_embedded_opmask_register_specifier(mask); 9229 if (merge) { 9230 attributes.reset_is_clear_context(); 9231 } 9232 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9233 emit_int8(0x5C); 9234 emit_operand(dst, src, 0); 9235 } 9236 9237 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9238 assert(VM_Version::supports_evex(), ""); 9239 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9240 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9241 attributes.set_is_evex_instruction(); 9242 attributes.set_embedded_opmask_register_specifier(mask); 9243 if (merge) { 9244 attributes.reset_is_clear_context(); 9245 } 9246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9247 emit_int16(0x5C, (0xC0 | encode)); 9248 } 9249 9250 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9251 InstructionMark im(this); 9252 assert(VM_Version::supports_evex(), ""); 9253 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9254 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9255 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9256 attributes.set_is_evex_instruction(); 9257 attributes.set_embedded_opmask_register_specifier(mask); 9258 if (merge) { 9259 attributes.reset_is_clear_context(); 9260 } 9261 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9262 emit_int8(0x5C); 9263 emit_operand(dst, src, 0); 9264 } 9265 9266 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9267 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9268 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9269 attributes.set_is_evex_instruction(); 9270 attributes.set_embedded_opmask_register_specifier(mask); 9271 if (merge) { 9272 attributes.reset_is_clear_context(); 9273 } 9274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9275 emit_int16((unsigned char)0xD5, (0xC0 | encode)); 9276 } 9277 9278 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9279 InstructionMark im(this); 9280 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9281 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9282 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9283 attributes.set_is_evex_instruction(); 9284 attributes.set_embedded_opmask_register_specifier(mask); 9285 if (merge) { 9286 attributes.reset_is_clear_context(); 9287 } 9288 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9289 emit_int8((unsigned char)0xD5); 9290 emit_operand(dst, src, 0); 9291 } 9292 9293 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9294 assert(VM_Version::supports_evex(), ""); 9295 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9296 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9297 attributes.set_is_evex_instruction(); 9298 attributes.set_embedded_opmask_register_specifier(mask); 9299 if (merge) { 9300 attributes.reset_is_clear_context(); 9301 } 9302 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9303 emit_int16(0x40, (0xC0 | encode)); 9304 } 9305 9306 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9307 InstructionMark im(this); 9308 assert(VM_Version::supports_evex(), ""); 9309 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9310 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9311 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9312 attributes.set_is_evex_instruction(); 9313 attributes.set_embedded_opmask_register_specifier(mask); 9314 if (merge) { 9315 attributes.reset_is_clear_context(); 9316 } 9317 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9318 emit_int8(0x40); 9319 emit_operand(dst, src, 0); 9320 } 9321 9322 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9323 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9324 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9325 attributes.set_is_evex_instruction(); 9326 attributes.set_embedded_opmask_register_specifier(mask); 9327 if (merge) { 9328 attributes.reset_is_clear_context(); 9329 } 9330 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9331 emit_int16(0x40, (0xC0 | encode)); 9332 } 9333 9334 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9335 InstructionMark im(this); 9336 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9337 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9338 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9339 attributes.set_is_evex_instruction(); 9340 attributes.set_embedded_opmask_register_specifier(mask); 9341 if (merge) { 9342 attributes.reset_is_clear_context(); 9343 } 9344 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9345 emit_int8(0x40); 9346 emit_operand(dst, src, 0); 9347 } 9348 9349 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9350 assert(VM_Version::supports_evex(), ""); 9351 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9352 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9353 attributes.set_is_evex_instruction(); 9354 attributes.set_embedded_opmask_register_specifier(mask); 9355 if (merge) { 9356 attributes.reset_is_clear_context(); 9357 } 9358 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9359 emit_int16(0x59, (0xC0 | encode)); 9360 } 9361 9362 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9363 InstructionMark im(this); 9364 assert(VM_Version::supports_evex(), ""); 9365 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9366 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9367 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9368 attributes.set_is_evex_instruction(); 9369 attributes.set_embedded_opmask_register_specifier(mask); 9370 if (merge) { 9371 attributes.reset_is_clear_context(); 9372 } 9373 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9374 emit_int8(0x59); 9375 emit_operand(dst, src, 0); 9376 } 9377 9378 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9379 assert(VM_Version::supports_evex(), ""); 9380 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9381 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9382 attributes.set_is_evex_instruction(); 9383 attributes.set_embedded_opmask_register_specifier(mask); 9384 if (merge) { 9385 attributes.reset_is_clear_context(); 9386 } 9387 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9388 emit_int16(0x59, (0xC0 | encode)); 9389 } 9390 9391 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9392 InstructionMark im(this); 9393 assert(VM_Version::supports_evex(), ""); 9394 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9395 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9396 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9397 attributes.set_is_evex_instruction(); 9398 attributes.set_embedded_opmask_register_specifier(mask); 9399 if (merge) { 9400 attributes.reset_is_clear_context(); 9401 } 9402 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9403 emit_int8(0x59); 9404 emit_operand(dst, src, 0); 9405 } 9406 9407 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9408 assert(VM_Version::supports_evex(), ""); 9409 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9410 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9411 attributes.set_is_evex_instruction(); 9412 attributes.set_embedded_opmask_register_specifier(mask); 9413 if (merge) { 9414 attributes.reset_is_clear_context(); 9415 } 9416 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9417 emit_int16(0x51, (0xC0 | encode)); 9418 } 9419 9420 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9421 InstructionMark im(this); 9422 assert(VM_Version::supports_evex(), ""); 9423 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9424 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9425 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9426 attributes.set_is_evex_instruction(); 9427 attributes.set_embedded_opmask_register_specifier(mask); 9428 if (merge) { 9429 attributes.reset_is_clear_context(); 9430 } 9431 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9432 emit_int8(0x51); 9433 emit_operand(dst, src, 0); 9434 } 9435 9436 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9437 assert(VM_Version::supports_evex(), ""); 9438 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9439 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9440 attributes.set_is_evex_instruction(); 9441 attributes.set_embedded_opmask_register_specifier(mask); 9442 if (merge) { 9443 attributes.reset_is_clear_context(); 9444 } 9445 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9446 emit_int16(0x51, (0xC0 | encode)); 9447 } 9448 9449 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9450 InstructionMark im(this); 9451 assert(VM_Version::supports_evex(), ""); 9452 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9453 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9454 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9455 attributes.set_is_evex_instruction(); 9456 attributes.set_embedded_opmask_register_specifier(mask); 9457 if (merge) { 9458 attributes.reset_is_clear_context(); 9459 } 9460 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9461 emit_int8(0x51); 9462 emit_operand(dst, src, 0); 9463 } 9464 9465 9466 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9467 assert(VM_Version::supports_evex(), ""); 9468 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9469 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9470 attributes.set_is_evex_instruction(); 9471 attributes.set_embedded_opmask_register_specifier(mask); 9472 if (merge) { 9473 attributes.reset_is_clear_context(); 9474 } 9475 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9476 emit_int16(0x5E, (0xC0 | encode)); 9477 } 9478 9479 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9480 InstructionMark im(this); 9481 assert(VM_Version::supports_evex(), ""); 9482 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9483 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9484 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9485 attributes.set_is_evex_instruction(); 9486 attributes.set_embedded_opmask_register_specifier(mask); 9487 if (merge) { 9488 attributes.reset_is_clear_context(); 9489 } 9490 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 9491 emit_int8(0x5E); 9492 emit_operand(dst, src, 0); 9493 } 9494 9495 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9496 assert(VM_Version::supports_evex(), ""); 9497 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9498 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9499 attributes.set_is_evex_instruction(); 9500 attributes.set_embedded_opmask_register_specifier(mask); 9501 if (merge) { 9502 attributes.reset_is_clear_context(); 9503 } 9504 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9505 emit_int16(0x5E, (0xC0 | encode)); 9506 } 9507 9508 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9509 InstructionMark im(this); 9510 assert(VM_Version::supports_evex(), ""); 9511 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9512 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9513 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9514 attributes.set_is_evex_instruction(); 9515 attributes.set_embedded_opmask_register_specifier(mask); 9516 if (merge) { 9517 attributes.reset_is_clear_context(); 9518 } 9519 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9520 emit_int8(0x5E); 9521 emit_operand(dst, src, 0); 9522 } 9523 9524 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9525 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9526 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9527 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9528 attributes.set_is_evex_instruction(); 9529 attributes.set_embedded_opmask_register_specifier(mask); 9530 if (merge) { 9531 attributes.reset_is_clear_context(); 9532 } 9533 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9534 emit_int16(0x1C, (0xC0 | encode)); 9535 } 9536 9537 9538 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 9539 InstructionMark im(this); 9540 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9541 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9542 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9543 attributes.set_is_evex_instruction(); 9544 attributes.set_embedded_opmask_register_specifier(mask); 9545 if (merge) { 9546 attributes.reset_is_clear_context(); 9547 } 9548 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9549 emit_int8(0x1C); 9550 emit_operand(dst, src, 0); 9551 } 9552 9553 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9554 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9555 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9556 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9557 attributes.set_is_evex_instruction(); 9558 attributes.set_embedded_opmask_register_specifier(mask); 9559 if (merge) { 9560 attributes.reset_is_clear_context(); 9561 } 9562 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9563 emit_int16(0x1D, (0xC0 | encode)); 9564 } 9565 9566 9567 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 9568 InstructionMark im(this); 9569 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9570 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9571 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9572 attributes.set_is_evex_instruction(); 9573 attributes.set_embedded_opmask_register_specifier(mask); 9574 if (merge) { 9575 attributes.reset_is_clear_context(); 9576 } 9577 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9578 emit_int8(0x1D); 9579 emit_operand(dst, src, 0); 9580 } 9581 9582 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9583 assert(VM_Version::supports_evex(), ""); 9584 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9585 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9586 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9587 attributes.set_is_evex_instruction(); 9588 attributes.set_embedded_opmask_register_specifier(mask); 9589 if (merge) { 9590 attributes.reset_is_clear_context(); 9591 } 9592 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9593 emit_int16(0x1E, (0xC0 | encode)); 9594 } 9595 9596 9597 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 9598 InstructionMark im(this); 9599 assert(VM_Version::supports_evex(), ""); 9600 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9601 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9602 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9603 attributes.set_is_evex_instruction(); 9604 attributes.set_embedded_opmask_register_specifier(mask); 9605 if (merge) { 9606 attributes.reset_is_clear_context(); 9607 } 9608 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9609 emit_int8(0x1E); 9610 emit_operand(dst, src, 0); 9611 } 9612 9613 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 9614 assert(VM_Version::supports_evex(), ""); 9615 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9616 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9617 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9618 attributes.set_is_evex_instruction(); 9619 attributes.set_embedded_opmask_register_specifier(mask); 9620 if (merge) { 9621 attributes.reset_is_clear_context(); 9622 } 9623 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9624 emit_int16(0x1F, (0xC0 | encode)); 9625 } 9626 9627 9628 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { 9629 InstructionMark im(this); 9630 assert(VM_Version::supports_evex(), ""); 9631 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9632 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9633 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9634 attributes.set_is_evex_instruction(); 9635 attributes.set_embedded_opmask_register_specifier(mask); 9636 if (merge) { 9637 attributes.reset_is_clear_context(); 9638 } 9639 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9640 emit_int8(0x1F); 9641 emit_operand(dst, src, 0); 9642 } 9643 9644 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9645 assert(VM_Version::supports_evex(), ""); 9646 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9647 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9648 attributes.set_is_evex_instruction(); 9649 attributes.set_embedded_opmask_register_specifier(mask); 9650 if (merge) { 9651 attributes.reset_is_clear_context(); 9652 } 9653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9654 emit_int16((unsigned char)0xA8, (0xC0 | encode)); 9655 } 9656 9657 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9658 InstructionMark im(this); 9659 assert(VM_Version::supports_evex(), ""); 9660 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9661 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9662 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9663 attributes.set_is_evex_instruction(); 9664 attributes.set_embedded_opmask_register_specifier(mask); 9665 if (merge) { 9666 attributes.reset_is_clear_context(); 9667 } 9668 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9669 emit_int8((unsigned char)0xA8); 9670 emit_operand(dst, src, 0); 9671 } 9672 9673 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9674 assert(VM_Version::supports_evex(), ""); 9675 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9676 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9677 attributes.set_is_evex_instruction(); 9678 attributes.set_embedded_opmask_register_specifier(mask); 9679 if (merge) { 9680 attributes.reset_is_clear_context(); 9681 } 9682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9683 emit_int16((unsigned char)0xA8, (0xC0 | encode)); 9684 } 9685 9686 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9687 InstructionMark im(this); 9688 assert(VM_Version::supports_evex(), ""); 9689 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9690 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); 9691 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit); 9692 attributes.set_is_evex_instruction(); 9693 attributes.set_embedded_opmask_register_specifier(mask); 9694 if (merge) { 9695 attributes.reset_is_clear_context(); 9696 } 9697 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9698 emit_int8((unsigned char)0xA8); 9699 emit_operand(dst, src, 0); 9700 } 9701 9702 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9703 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9704 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9705 attributes.set_is_evex_instruction(); 9706 attributes.set_embedded_opmask_register_specifier(mask); 9707 if (merge) { 9708 attributes.reset_is_clear_context(); 9709 } 9710 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9711 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 9712 } 9713 9714 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9715 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9716 InstructionMark im(this); 9717 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9718 attributes.set_is_evex_instruction(); 9719 attributes.set_embedded_opmask_register_specifier(mask); 9720 if (merge) { 9721 attributes.reset_is_clear_context(); 9722 } 9723 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9724 emit_int8((unsigned char)0x8D); 9725 emit_operand(dst, src, 0); 9726 } 9727 9728 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9729 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9730 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9731 attributes.set_is_evex_instruction(); 9732 attributes.set_embedded_opmask_register_specifier(mask); 9733 if (merge) { 9734 attributes.reset_is_clear_context(); 9735 } 9736 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9737 emit_int16((unsigned char)0x8D, (0xC0 | encode)); 9738 } 9739 9740 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9741 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9742 InstructionMark im(this); 9743 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9744 attributes.set_is_evex_instruction(); 9745 attributes.set_embedded_opmask_register_specifier(mask); 9746 if (merge) { 9747 attributes.reset_is_clear_context(); 9748 } 9749 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9750 emit_int8((unsigned char)0x8D); 9751 emit_operand(dst, src, 0); 9752 } 9753 9754 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9755 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 9756 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9757 attributes.set_is_evex_instruction(); 9758 attributes.set_embedded_opmask_register_specifier(mask); 9759 if (merge) { 9760 attributes.reset_is_clear_context(); 9761 } 9762 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9763 emit_int16(0x36, (0xC0 | encode)); 9764 } 9765 9766 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9767 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 9768 InstructionMark im(this); 9769 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9770 attributes.set_is_evex_instruction(); 9771 attributes.set_embedded_opmask_register_specifier(mask); 9772 if (merge) { 9773 attributes.reset_is_clear_context(); 9774 } 9775 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9776 emit_int8(0x36); 9777 emit_operand(dst, src, 0); 9778 } 9779 9780 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9781 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 9782 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9783 attributes.set_is_evex_instruction(); 9784 attributes.set_embedded_opmask_register_specifier(mask); 9785 if (merge) { 9786 attributes.reset_is_clear_context(); 9787 } 9788 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9789 emit_int16(0x36, (0xC0 | encode)); 9790 } 9791 9792 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9793 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, ""); 9794 InstructionMark im(this); 9795 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9796 attributes.set_is_evex_instruction(); 9797 attributes.set_embedded_opmask_register_specifier(mask); 9798 if (merge) { 9799 attributes.reset_is_clear_context(); 9800 } 9801 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 9802 emit_int8(0x36); 9803 emit_operand(dst, src, 0); 9804 } 9805 9806 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9807 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9808 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9809 attributes.set_is_evex_instruction(); 9810 attributes.set_embedded_opmask_register_specifier(mask); 9811 if (merge) { 9812 attributes.reset_is_clear_context(); 9813 } 9814 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9815 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9816 } 9817 9818 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9819 assert(VM_Version::supports_evex(), ""); 9820 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9821 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9822 attributes.set_is_evex_instruction(); 9823 attributes.set_embedded_opmask_register_specifier(mask); 9824 if (merge) { 9825 attributes.reset_is_clear_context(); 9826 } 9827 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9828 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9829 } 9830 9831 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9832 assert(VM_Version::supports_evex(), ""); 9833 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9834 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9835 attributes.set_is_evex_instruction(); 9836 attributes.set_embedded_opmask_register_specifier(mask); 9837 if (merge) { 9838 attributes.reset_is_clear_context(); 9839 } 9840 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9841 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9842 } 9843 9844 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9845 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9846 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9847 attributes.set_is_evex_instruction(); 9848 attributes.set_embedded_opmask_register_specifier(mask); 9849 if (merge) { 9850 attributes.reset_is_clear_context(); 9851 } 9852 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9853 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9854 } 9855 9856 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9857 assert(VM_Version::supports_evex(), ""); 9858 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9859 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9860 attributes.set_is_evex_instruction(); 9861 attributes.set_embedded_opmask_register_specifier(mask); 9862 if (merge) { 9863 attributes.reset_is_clear_context(); 9864 } 9865 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9866 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9867 } 9868 9869 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9870 assert(VM_Version::supports_evex(), ""); 9871 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9872 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9873 attributes.set_is_evex_instruction(); 9874 attributes.set_embedded_opmask_register_specifier(mask); 9875 if (merge) { 9876 attributes.reset_is_clear_context(); 9877 } 9878 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9879 emit_int24(0x73, (0xC0 | encode), shift & 0xFF); 9880 } 9881 9882 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9883 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9884 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9885 attributes.set_is_evex_instruction(); 9886 attributes.set_embedded_opmask_register_specifier(mask); 9887 if (merge) { 9888 attributes.reset_is_clear_context(); 9889 } 9890 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9891 emit_int24(0x71, (0xC0 | encode), shift & 0xFF); 9892 } 9893 9894 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9895 assert(VM_Version::supports_evex(), ""); 9896 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9897 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9898 attributes.set_is_evex_instruction(); 9899 attributes.set_embedded_opmask_register_specifier(mask); 9900 if (merge) { 9901 attributes.reset_is_clear_context(); 9902 } 9903 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9904 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9905 } 9906 9907 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 9908 assert(VM_Version::supports_evex(), ""); 9909 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9910 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9911 attributes.set_is_evex_instruction(); 9912 attributes.set_embedded_opmask_register_specifier(mask); 9913 if (merge) { 9914 attributes.reset_is_clear_context(); 9915 } 9916 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9917 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 9918 } 9919 9920 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9921 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9922 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9923 attributes.set_is_evex_instruction(); 9924 attributes.set_embedded_opmask_register_specifier(mask); 9925 if (merge) { 9926 attributes.reset_is_clear_context(); 9927 } 9928 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9929 emit_int16((unsigned char)0xF1, (0xC0 | encode)); 9930 } 9931 9932 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9933 assert(VM_Version::supports_evex(), ""); 9934 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9935 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9936 attributes.set_is_evex_instruction(); 9937 attributes.set_embedded_opmask_register_specifier(mask); 9938 if (merge) { 9939 attributes.reset_is_clear_context(); 9940 } 9941 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9942 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 9943 } 9944 9945 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9946 assert(VM_Version::supports_evex(), ""); 9947 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9948 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9949 attributes.set_is_evex_instruction(); 9950 attributes.set_embedded_opmask_register_specifier(mask); 9951 if (merge) { 9952 attributes.reset_is_clear_context(); 9953 } 9954 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9955 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 9956 } 9957 9958 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9959 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9960 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9961 attributes.set_is_evex_instruction(); 9962 attributes.set_embedded_opmask_register_specifier(mask); 9963 if (merge) { 9964 attributes.reset_is_clear_context(); 9965 } 9966 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9967 emit_int16((unsigned char)0xD1, (0xC0 | encode)); 9968 } 9969 9970 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9971 assert(VM_Version::supports_evex(), ""); 9972 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9973 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9974 attributes.set_is_evex_instruction(); 9975 attributes.set_embedded_opmask_register_specifier(mask); 9976 if (merge) { 9977 attributes.reset_is_clear_context(); 9978 } 9979 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9980 emit_int16((unsigned char)0xD2, (0xC0 | encode)); 9981 } 9982 9983 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9984 assert(VM_Version::supports_evex(), ""); 9985 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 9986 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9987 attributes.set_is_evex_instruction(); 9988 attributes.set_embedded_opmask_register_specifier(mask); 9989 if (merge) { 9990 attributes.reset_is_clear_context(); 9991 } 9992 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 9993 emit_int16((unsigned char)0xD3, (0xC0 | encode)); 9994 } 9995 9996 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9997 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 9998 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 9999 attributes.set_is_evex_instruction(); 10000 attributes.set_embedded_opmask_register_specifier(mask); 10001 if (merge) { 10002 attributes.reset_is_clear_context(); 10003 } 10004 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10005 emit_int16((unsigned char)0xE1, (0xC0 | encode)); 10006 } 10007 10008 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10009 assert(VM_Version::supports_evex(), ""); 10010 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10011 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10012 attributes.set_is_evex_instruction(); 10013 attributes.set_embedded_opmask_register_specifier(mask); 10014 if (merge) { 10015 attributes.reset_is_clear_context(); 10016 } 10017 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10018 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 10019 } 10020 10021 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10022 assert(VM_Version::supports_evex(), ""); 10023 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10024 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10025 attributes.set_is_evex_instruction(); 10026 attributes.set_embedded_opmask_register_specifier(mask); 10027 if (merge) { 10028 attributes.reset_is_clear_context(); 10029 } 10030 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10031 emit_int16((unsigned char)0xE2, (0xC0 | encode)); 10032 } 10033 10034 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10035 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10036 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10037 attributes.set_is_evex_instruction(); 10038 attributes.set_embedded_opmask_register_specifier(mask); 10039 if (merge) { 10040 attributes.reset_is_clear_context(); 10041 } 10042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10043 emit_int16(0x12, (0xC0 | encode)); 10044 } 10045 10046 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10047 assert(VM_Version::supports_evex(), ""); 10048 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10049 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10050 attributes.set_is_evex_instruction(); 10051 attributes.set_embedded_opmask_register_specifier(mask); 10052 if (merge) { 10053 attributes.reset_is_clear_context(); 10054 } 10055 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10056 emit_int16(0x47, (0xC0 | encode)); 10057 } 10058 10059 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10060 assert(VM_Version::supports_evex(), ""); 10061 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10062 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10063 attributes.set_is_evex_instruction(); 10064 attributes.set_embedded_opmask_register_specifier(mask); 10065 if (merge) { 10066 attributes.reset_is_clear_context(); 10067 } 10068 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10069 emit_int16(0x47, (0xC0 | encode)); 10070 } 10071 10072 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10073 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10074 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10075 attributes.set_is_evex_instruction(); 10076 attributes.set_embedded_opmask_register_specifier(mask); 10077 if (merge) { 10078 attributes.reset_is_clear_context(); 10079 } 10080 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10081 emit_int16(0x10, (0xC0 | encode)); 10082 } 10083 10084 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10085 assert(VM_Version::supports_evex(), ""); 10086 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10088 attributes.set_is_evex_instruction(); 10089 attributes.set_embedded_opmask_register_specifier(mask); 10090 if (merge) { 10091 attributes.reset_is_clear_context(); 10092 } 10093 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10094 emit_int16(0x45, (0xC0 | encode)); 10095 } 10096 10097 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10098 assert(VM_Version::supports_evex(), ""); 10099 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10100 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10101 attributes.set_is_evex_instruction(); 10102 attributes.set_embedded_opmask_register_specifier(mask); 10103 if (merge) { 10104 attributes.reset_is_clear_context(); 10105 } 10106 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10107 emit_int16(0x45, (0xC0 | encode)); 10108 } 10109 10110 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10111 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10112 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10113 attributes.set_is_evex_instruction(); 10114 attributes.set_embedded_opmask_register_specifier(mask); 10115 if (merge) { 10116 attributes.reset_is_clear_context(); 10117 } 10118 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10119 emit_int16(0x11, (0xC0 | encode)); 10120 } 10121 10122 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10123 assert(VM_Version::supports_evex(), ""); 10124 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10126 attributes.set_is_evex_instruction(); 10127 attributes.set_embedded_opmask_register_specifier(mask); 10128 if (merge) { 10129 attributes.reset_is_clear_context(); 10130 } 10131 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10132 emit_int16(0x46, (0xC0 | encode)); 10133 } 10134 10135 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10136 assert(VM_Version::supports_evex(), ""); 10137 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10138 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10139 attributes.set_is_evex_instruction(); 10140 attributes.set_embedded_opmask_register_specifier(mask); 10141 if (merge) { 10142 attributes.reset_is_clear_context(); 10143 } 10144 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10145 emit_int16(0x46, (0xC0 | encode)); 10146 } 10147 10148 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10149 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10150 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10151 attributes.set_is_evex_instruction(); 10152 attributes.set_embedded_opmask_register_specifier(mask); 10153 if (merge) { 10154 attributes.reset_is_clear_context(); 10155 } 10156 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10157 emit_int16(0x38, (0xC0 | encode)); 10158 } 10159 10160 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10161 assert(VM_Version::supports_avx512bw(), ""); 10162 InstructionMark im(this); 10163 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10164 attributes.set_is_evex_instruction(); 10165 attributes.set_embedded_opmask_register_specifier(mask); 10166 if (merge) { 10167 attributes.reset_is_clear_context(); 10168 } 10169 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10170 emit_int8(0x38); 10171 emit_operand(dst, src, 0); 10172 } 10173 10174 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10175 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10176 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10177 attributes.set_is_evex_instruction(); 10178 attributes.set_embedded_opmask_register_specifier(mask); 10179 if (merge) { 10180 attributes.reset_is_clear_context(); 10181 } 10182 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10183 emit_int16((unsigned char)0xEA, (0xC0 | encode)); 10184 } 10185 10186 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10187 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10188 InstructionMark im(this); 10189 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10190 attributes.set_is_evex_instruction(); 10191 attributes.set_embedded_opmask_register_specifier(mask); 10192 if (merge) { 10193 attributes.reset_is_clear_context(); 10194 } 10195 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10196 emit_int8((unsigned char)0xEA); 10197 emit_operand(dst, src, 0); 10198 } 10199 10200 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10201 assert(VM_Version::supports_evex(), ""); 10202 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10203 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10204 attributes.set_is_evex_instruction(); 10205 attributes.set_embedded_opmask_register_specifier(mask); 10206 if (merge) { 10207 attributes.reset_is_clear_context(); 10208 } 10209 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10210 emit_int16(0x39, (0xC0 | encode)); 10211 } 10212 10213 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10214 assert(VM_Version::supports_evex(), ""); 10215 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10216 InstructionMark im(this); 10217 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10218 attributes.set_is_evex_instruction(); 10219 attributes.set_embedded_opmask_register_specifier(mask); 10220 if (merge) { 10221 attributes.reset_is_clear_context(); 10222 } 10223 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10224 emit_int8(0x39); 10225 emit_operand(dst, src, 0); 10226 } 10227 10228 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10229 assert(VM_Version::supports_evex(), ""); 10230 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10231 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10232 attributes.set_is_evex_instruction(); 10233 attributes.set_embedded_opmask_register_specifier(mask); 10234 if (merge) { 10235 attributes.reset_is_clear_context(); 10236 } 10237 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10238 emit_int16(0x39, (0xC0 | encode)); 10239 } 10240 10241 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10242 assert(VM_Version::supports_evex(), ""); 10243 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10244 InstructionMark im(this); 10245 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10246 attributes.set_is_evex_instruction(); 10247 attributes.set_embedded_opmask_register_specifier(mask); 10248 if (merge) { 10249 attributes.reset_is_clear_context(); 10250 } 10251 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10252 emit_int8(0x39); 10253 emit_operand(dst, src, 0); 10254 } 10255 10256 10257 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10258 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10259 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10260 attributes.set_is_evex_instruction(); 10261 attributes.set_embedded_opmask_register_specifier(mask); 10262 if (merge) { 10263 attributes.reset_is_clear_context(); 10264 } 10265 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10266 emit_int16(0x3C, (0xC0 | encode)); 10267 } 10268 10269 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10270 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10271 InstructionMark im(this); 10272 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10273 attributes.set_is_evex_instruction(); 10274 attributes.set_embedded_opmask_register_specifier(mask); 10275 if (merge) { 10276 attributes.reset_is_clear_context(); 10277 } 10278 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10279 emit_int8(0x3C); 10280 emit_operand(dst, src, 0); 10281 } 10282 10283 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10284 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10285 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10286 attributes.set_is_evex_instruction(); 10287 attributes.set_embedded_opmask_register_specifier(mask); 10288 if (merge) { 10289 attributes.reset_is_clear_context(); 10290 } 10291 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10292 emit_int16((unsigned char)0xEE, (0xC0 | encode)); 10293 } 10294 10295 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10296 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), ""); 10297 InstructionMark im(this); 10298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10299 attributes.set_is_evex_instruction(); 10300 attributes.set_embedded_opmask_register_specifier(mask); 10301 if (merge) { 10302 attributes.reset_is_clear_context(); 10303 } 10304 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 10305 emit_int8((unsigned char)0xEE); 10306 emit_operand(dst, src, 0); 10307 } 10308 10309 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10310 assert(VM_Version::supports_evex(), ""); 10311 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10312 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10313 attributes.set_is_evex_instruction(); 10314 attributes.set_embedded_opmask_register_specifier(mask); 10315 if (merge) { 10316 attributes.reset_is_clear_context(); 10317 } 10318 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10319 emit_int16(0x3D, (0xC0 | encode)); 10320 } 10321 10322 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10323 assert(VM_Version::supports_evex(), ""); 10324 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10325 InstructionMark im(this); 10326 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10327 attributes.set_is_evex_instruction(); 10328 attributes.set_embedded_opmask_register_specifier(mask); 10329 if (merge) { 10330 attributes.reset_is_clear_context(); 10331 } 10332 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10333 emit_int8(0x3D); 10334 emit_operand(dst, src, 0); 10335 } 10336 10337 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10338 assert(VM_Version::supports_evex(), ""); 10339 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10340 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10341 attributes.set_is_evex_instruction(); 10342 attributes.set_embedded_opmask_register_specifier(mask); 10343 if (merge) { 10344 attributes.reset_is_clear_context(); 10345 } 10346 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10347 emit_int16(0x3D, (0xC0 | encode)); 10348 } 10349 10350 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10351 assert(VM_Version::supports_evex(), ""); 10352 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 10353 InstructionMark im(this); 10354 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10355 attributes.set_is_evex_instruction(); 10356 attributes.set_embedded_opmask_register_specifier(mask); 10357 if (merge) { 10358 attributes.reset_is_clear_context(); 10359 } 10360 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10361 emit_int8(0x3D); 10362 emit_operand(dst, src, 0); 10363 } 10364 10365 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) { 10366 assert(VM_Version::supports_evex(), "requires EVEX support"); 10367 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10368 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10369 attributes.set_is_evex_instruction(); 10370 attributes.set_embedded_opmask_register_specifier(mask); 10371 if (merge) { 10372 attributes.reset_is_clear_context(); 10373 } 10374 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10375 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8); 10376 } 10377 10378 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) { 10379 assert(VM_Version::supports_evex(), "requires EVEX support"); 10380 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10381 assert(dst != xnoreg, "sanity"); 10382 InstructionMark im(this); 10383 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10384 attributes.set_is_evex_instruction(); 10385 attributes.set_embedded_opmask_register_specifier(mask); 10386 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 10387 if (merge) { 10388 attributes.reset_is_clear_context(); 10389 } 10390 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10391 emit_int8(0x25); 10392 emit_operand(dst, src3, 1); 10393 emit_int8(imm8); 10394 } 10395 10396 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) { 10397 assert(VM_Version::supports_evex(), "requires EVEX support"); 10398 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10399 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10400 attributes.set_is_evex_instruction(); 10401 attributes.set_embedded_opmask_register_specifier(mask); 10402 if (merge) { 10403 attributes.reset_is_clear_context(); 10404 } 10405 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10406 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8); 10407 } 10408 10409 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) { 10410 assert(VM_Version::supports_evex(), "requires EVEX support"); 10411 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support"); 10412 assert(dst != xnoreg, "sanity"); 10413 InstructionMark im(this); 10414 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10415 attributes.set_is_evex_instruction(); 10416 attributes.set_embedded_opmask_register_specifier(mask); 10417 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit); 10418 if (merge) { 10419 attributes.reset_is_clear_context(); 10420 } 10421 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10422 emit_int8(0x25); 10423 emit_operand(dst, src3, 1); 10424 emit_int8(imm8); 10425 } 10426 10427 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) { 10428 assert(VM_Version::supports_gfni(), ""); 10429 assert(VM_Version::supports_sse(), ""); 10430 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 10431 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10432 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8); 10433 } 10434 10435 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) { 10436 assert(VM_Version::supports_gfni(), "requires GFNI support"); 10437 assert(VM_Version::supports_sse(), ""); 10438 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10439 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10440 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8); 10441 } 10442 10443 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL 10444 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) { 10445 assert(UseAVX >= 2, ""); 10446 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10447 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10448 emit_int16(0x58, (0xC0 | encode)); 10449 } 10450 10451 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) { 10452 assert(VM_Version::supports_avx2(), ""); 10453 assert(dst != xnoreg, "sanity"); 10454 InstructionMark im(this); 10455 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10456 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10457 // swap src<->dst for encoding 10458 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10459 emit_int8(0x58); 10460 emit_operand(dst, src, 0); 10461 } 10462 10463 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL 10464 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) { 10465 assert(VM_Version::supports_avx2(), ""); 10466 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10467 attributes.set_rex_vex_w_reverted(); 10468 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10469 emit_int16(0x59, (0xC0 | encode)); 10470 } 10471 10472 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) { 10473 assert(VM_Version::supports_avx2(), ""); 10474 assert(dst != xnoreg, "sanity"); 10475 InstructionMark im(this); 10476 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10477 attributes.set_rex_vex_w_reverted(); 10478 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 10479 // swap src<->dst for encoding 10480 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10481 emit_int8(0x59); 10482 emit_operand(dst, src, 0); 10483 } 10484 10485 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) { 10486 assert(vector_len != Assembler::AVX_128bit, ""); 10487 assert(VM_Version::supports_evex(), ""); 10488 assert(dst != xnoreg, "sanity"); 10489 InstructionMark im(this); 10490 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10491 attributes.set_rex_vex_w_reverted(); 10492 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10493 // swap src<->dst for encoding 10494 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10495 emit_int8(0x5A); 10496 emit_operand(dst, src, 0); 10497 } 10498 10499 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) { 10500 assert(vector_len != Assembler::AVX_128bit, ""); 10501 assert(VM_Version::supports_avx512dq(), ""); 10502 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10503 attributes.set_rex_vex_w_reverted(); 10504 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10505 emit_int16(0x5A, (0xC0 | encode)); 10506 } 10507 10508 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) { 10509 assert(vector_len != Assembler::AVX_128bit, ""); 10510 assert(VM_Version::supports_avx512dq(), ""); 10511 assert(dst != xnoreg, "sanity"); 10512 InstructionMark im(this); 10513 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10514 attributes.set_rex_vex_w_reverted(); 10515 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit); 10516 // swap src<->dst for encoding 10517 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10518 emit_int8(0x5A); 10519 emit_operand(dst, src, 0); 10520 } 10521 10522 // scalar single/double precision replicate 10523 10524 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL 10525 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) { 10526 assert(VM_Version::supports_avx2(), ""); 10527 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10528 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10529 emit_int16(0x18, (0xC0 | encode)); 10530 } 10531 10532 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) { 10533 assert(VM_Version::supports_avx(), ""); 10534 assert(dst != xnoreg, "sanity"); 10535 InstructionMark im(this); 10536 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10537 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10538 // swap src<->dst for encoding 10539 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10540 emit_int8(0x18); 10541 emit_operand(dst, src, 0); 10542 } 10543 10544 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL 10545 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) { 10546 assert(VM_Version::supports_avx2(), ""); 10547 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, ""); 10548 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10549 attributes.set_rex_vex_w_reverted(); 10550 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10551 emit_int16(0x19, (0xC0 | encode)); 10552 } 10553 10554 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) { 10555 assert(VM_Version::supports_avx(), ""); 10556 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, ""); 10557 assert(dst != xnoreg, "sanity"); 10558 InstructionMark im(this); 10559 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10560 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 10561 attributes.set_rex_vex_w_reverted(); 10562 // swap src<->dst for encoding 10563 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10564 emit_int8(0x19); 10565 emit_operand(dst, src, 0); 10566 } 10567 10568 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) { 10569 assert(VM_Version::supports_avx(), ""); 10570 assert(vector_len == AVX_256bit, ""); 10571 assert(dst != xnoreg, "sanity"); 10572 InstructionMark im(this); 10573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10574 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit); 10575 // swap src<->dst for encoding 10576 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10577 emit_int8(0x1A); 10578 emit_operand(dst, src, 0); 10579 } 10580 10581 // gpr source broadcast forms 10582 10583 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 10584 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) { 10585 assert(VM_Version::supports_avx512bw(), ""); 10586 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10587 attributes.set_is_evex_instruction(); 10588 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10589 emit_int16(0x7A, (0xC0 | encode)); 10590 } 10591 10592 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL 10593 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) { 10594 assert(VM_Version::supports_avx512bw(), ""); 10595 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); 10596 attributes.set_is_evex_instruction(); 10597 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10598 emit_int16(0x7B, (0xC0 | encode)); 10599 } 10600 10601 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL 10602 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) { 10603 assert(VM_Version::supports_evex(), ""); 10604 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10605 attributes.set_is_evex_instruction(); 10606 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10607 emit_int16(0x7C, (0xC0 | encode)); 10608 } 10609 10610 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL 10611 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) { 10612 assert(VM_Version::supports_evex(), ""); 10613 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10614 attributes.set_is_evex_instruction(); 10615 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10616 emit_int16(0x7C, (0xC0 | encode)); 10617 } 10618 10619 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 10620 assert(VM_Version::supports_avx2(), ""); 10621 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 10622 assert(dst != xnoreg, "sanity"); 10623 assert(src.isxmmindex(),"expected to be xmm index"); 10624 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10625 InstructionMark im(this); 10626 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 10627 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10628 emit_int8((unsigned char)0x90); 10629 emit_operand(dst, src, 0); 10630 } 10631 10632 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 10633 assert(VM_Version::supports_avx2(), ""); 10634 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 10635 assert(dst != xnoreg, "sanity"); 10636 assert(src.isxmmindex(),"expected to be xmm index"); 10637 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10638 InstructionMark im(this); 10639 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 10640 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10641 emit_int8((unsigned char)0x90); 10642 emit_operand(dst, src, 0); 10643 } 10644 10645 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 10646 assert(VM_Version::supports_avx2(), ""); 10647 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 10648 assert(dst != xnoreg, "sanity"); 10649 assert(src.isxmmindex(),"expected to be xmm index"); 10650 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10651 InstructionMark im(this); 10652 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 10653 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10654 emit_int8((unsigned char)0x92); 10655 emit_operand(dst, src, 0); 10656 } 10657 10658 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) { 10659 assert(VM_Version::supports_avx2(), ""); 10660 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, ""); 10661 assert(dst != xnoreg, "sanity"); 10662 assert(src.isxmmindex(),"expected to be xmm index"); 10663 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10664 InstructionMark im(this); 10665 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true); 10666 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10667 emit_int8((unsigned char)0x92); 10668 emit_operand(dst, src, 0); 10669 } 10670 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 10671 assert(VM_Version::supports_evex(), ""); 10672 assert(dst != xnoreg, "sanity"); 10673 assert(src.isxmmindex(),"expected to be xmm index"); 10674 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10675 assert(mask != k0, "instruction will #UD if mask is in k0"); 10676 InstructionMark im(this); 10677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10678 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10679 attributes.reset_is_clear_context(); 10680 attributes.set_embedded_opmask_register_specifier(mask); 10681 attributes.set_is_evex_instruction(); 10682 // swap src<->dst for encoding 10683 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10684 emit_int8((unsigned char)0x90); 10685 emit_operand(dst, src, 0); 10686 } 10687 10688 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) { 10689 assert(VM_Version::supports_evex(), ""); 10690 assert(dst != xnoreg, "sanity"); 10691 assert(src.isxmmindex(),"expected to be xmm index"); 10692 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10693 assert(mask != k0, "instruction will #UD if mask is in k0"); 10694 InstructionMark im(this); 10695 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10696 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10697 attributes.reset_is_clear_context(); 10698 attributes.set_embedded_opmask_register_specifier(mask); 10699 attributes.set_is_evex_instruction(); 10700 // swap src<->dst for encoding 10701 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10702 emit_int8((unsigned char)0x90); 10703 emit_operand(dst, src, 0); 10704 } 10705 10706 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) { 10707 assert(VM_Version::supports_evex(), ""); 10708 assert(dst != xnoreg, "sanity"); 10709 assert(src.isxmmindex(),"expected to be xmm index"); 10710 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10711 assert(mask != k0, "instruction will #UD if mask is in k0"); 10712 InstructionMark im(this); 10713 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10714 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10715 attributes.reset_is_clear_context(); 10716 attributes.set_embedded_opmask_register_specifier(mask); 10717 attributes.set_is_evex_instruction(); 10718 // swap src<->dst for encoding 10719 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10720 emit_int8((unsigned char)0x92); 10721 emit_operand(dst, src, 0); 10722 } 10723 10724 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) { 10725 assert(VM_Version::supports_evex(), ""); 10726 assert(dst != xnoreg, "sanity"); 10727 assert(src.isxmmindex(),"expected to be xmm index"); 10728 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same"); 10729 assert(mask != k0, "instruction will #UD if mask is in k0"); 10730 InstructionMark im(this); 10731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10732 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10733 attributes.reset_is_clear_context(); 10734 attributes.set_embedded_opmask_register_specifier(mask); 10735 attributes.set_is_evex_instruction(); 10736 // swap src<->dst for encoding 10737 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10738 emit_int8((unsigned char)0x92); 10739 emit_operand(dst, src, 0); 10740 } 10741 10742 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) { 10743 assert(VM_Version::supports_evex(), ""); 10744 assert(mask != k0, "instruction will #UD if mask is in k0"); 10745 InstructionMark im(this); 10746 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10747 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10748 attributes.reset_is_clear_context(); 10749 attributes.set_embedded_opmask_register_specifier(mask); 10750 attributes.set_is_evex_instruction(); 10751 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10752 emit_int8((unsigned char)0xA0); 10753 emit_operand(src, dst, 0); 10754 } 10755 10756 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) { 10757 assert(VM_Version::supports_evex(), ""); 10758 assert(mask != k0, "instruction will #UD if mask is in k0"); 10759 InstructionMark im(this); 10760 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10761 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10762 attributes.reset_is_clear_context(); 10763 attributes.set_embedded_opmask_register_specifier(mask); 10764 attributes.set_is_evex_instruction(); 10765 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10766 emit_int8((unsigned char)0xA0); 10767 emit_operand(src, dst, 0); 10768 } 10769 10770 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) { 10771 assert(VM_Version::supports_evex(), ""); 10772 assert(mask != k0, "instruction will #UD if mask is in k0"); 10773 InstructionMark im(this); 10774 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10775 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10776 attributes.reset_is_clear_context(); 10777 attributes.set_embedded_opmask_register_specifier(mask); 10778 attributes.set_is_evex_instruction(); 10779 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10780 emit_int8((unsigned char)0xA2); 10781 emit_operand(src, dst, 0); 10782 } 10783 10784 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) { 10785 assert(VM_Version::supports_evex(), ""); 10786 assert(mask != k0, "instruction will #UD if mask is in k0"); 10787 InstructionMark im(this); 10788 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 10789 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit); 10790 attributes.reset_is_clear_context(); 10791 attributes.set_embedded_opmask_register_specifier(mask); 10792 attributes.set_is_evex_instruction(); 10793 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 10794 emit_int8((unsigned char)0xA2); 10795 emit_operand(src, dst, 0); 10796 } 10797 // Carry-Less Multiplication Quadword 10798 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) { 10799 assert(VM_Version::supports_clmul(), ""); 10800 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 10801 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10802 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 10803 } 10804 10805 // Carry-Less Multiplication Quadword 10806 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) { 10807 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), ""); 10808 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 10809 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10810 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 10811 } 10812 10813 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) { 10814 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support"); 10815 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 10816 attributes.set_is_evex_instruction(); 10817 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10818 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask); 10819 } 10820 10821 void Assembler::vzeroupper_uncached() { 10822 if (VM_Version::supports_vzeroupper()) { 10823 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 10824 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 10825 emit_int8(0x77); 10826 } 10827 } 10828 10829 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) { 10830 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib 10831 assert(VM_Version::supports_evex(), ""); 10832 assert(VM_Version::supports_avx512dq(), ""); 10833 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false); 10834 attributes.set_is_evex_instruction(); 10835 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10836 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8); 10837 } 10838 10839 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) { 10840 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib 10841 assert(VM_Version::supports_evex(), ""); 10842 assert(VM_Version::supports_avx512dq(), ""); 10843 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false); 10844 attributes.set_is_evex_instruction(); 10845 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 10846 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8); 10847 } 10848 10849 void Assembler::fld_x(Address adr) { 10850 InstructionMark im(this); 10851 emit_int8((unsigned char)0xDB); 10852 emit_operand32(rbp, adr, 0); 10853 } 10854 10855 void Assembler::fstp_x(Address adr) { 10856 InstructionMark im(this); 10857 emit_int8((unsigned char)0xDB); 10858 emit_operand32(rdi, adr, 0); 10859 } 10860 10861 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) { 10862 assert(reg->encoding() < 8, "no extended registers"); 10863 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers"); 10864 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length); 10865 } 10866 10867 #ifndef _LP64 10868 // 32bit only pieces of the assembler 10869 10870 void Assembler::emms() { 10871 NOT_LP64(assert(VM_Version::supports_mmx(), "")); 10872 emit_int16(0x0F, 0x77); 10873 } 10874 10875 void Assembler::vzeroupper() { 10876 vzeroupper_uncached(); 10877 } 10878 10879 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) { 10880 // NO PREFIX AS NEVER 64BIT 10881 InstructionMark im(this); 10882 emit_int16((unsigned char)0x81, (0xF8 | src1->encoding())); 10883 emit_data(imm32, rspec, 0); 10884 } 10885 10886 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) { 10887 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs 10888 InstructionMark im(this); 10889 emit_int8((unsigned char)0x81); 10890 emit_operand(rdi, src1, 4); 10891 emit_data(imm32, rspec, 0); 10892 } 10893 10894 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax, 10895 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded 10896 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise. 10897 void Assembler::cmpxchg8(Address adr) { 10898 InstructionMark im(this); 10899 emit_int16(0x0F, (unsigned char)0xC7); 10900 emit_operand(rcx, adr, 0); 10901 } 10902 10903 void Assembler::decl(Register dst) { 10904 // Don't use it directly. Use MacroAssembler::decrementl() instead. 10905 emit_int8(0x48 | dst->encoding()); 10906 } 10907 10908 // 64bit doesn't use the x87 10909 10910 void Assembler::emit_farith(int b1, int b2, int i) { 10911 assert(isByte(b1) && isByte(b2), "wrong opcode"); 10912 assert(0 <= i && i < 8, "illegal stack offset"); 10913 emit_int16(b1, b2 + i); 10914 } 10915 10916 void Assembler::fabs() { 10917 emit_int16((unsigned char)0xD9, (unsigned char)0xE1); 10918 } 10919 10920 void Assembler::fadd(int i) { 10921 emit_farith(0xD8, 0xC0, i); 10922 } 10923 10924 void Assembler::fadd_d(Address src) { 10925 InstructionMark im(this); 10926 emit_int8((unsigned char)0xDC); 10927 emit_operand32(rax, src, 0); 10928 } 10929 10930 void Assembler::fadd_s(Address src) { 10931 InstructionMark im(this); 10932 emit_int8((unsigned char)0xD8); 10933 emit_operand32(rax, src, 0); 10934 } 10935 10936 void Assembler::fadda(int i) { 10937 emit_farith(0xDC, 0xC0, i); 10938 } 10939 10940 void Assembler::faddp(int i) { 10941 emit_farith(0xDE, 0xC0, i); 10942 } 10943 10944 void Assembler::fchs() { 10945 emit_int16((unsigned char)0xD9, (unsigned char)0xE0); 10946 } 10947 10948 void Assembler::fcom(int i) { 10949 emit_farith(0xD8, 0xD0, i); 10950 } 10951 10952 void Assembler::fcomp(int i) { 10953 emit_farith(0xD8, 0xD8, i); 10954 } 10955 10956 void Assembler::fcomp_d(Address src) { 10957 InstructionMark im(this); 10958 emit_int8((unsigned char)0xDC); 10959 emit_operand32(rbx, src, 0); 10960 } 10961 10962 void Assembler::fcomp_s(Address src) { 10963 InstructionMark im(this); 10964 emit_int8((unsigned char)0xD8); 10965 emit_operand32(rbx, src, 0); 10966 } 10967 10968 void Assembler::fcompp() { 10969 emit_int16((unsigned char)0xDE, (unsigned char)0xD9); 10970 } 10971 10972 void Assembler::fcos() { 10973 emit_int16((unsigned char)0xD9, (unsigned char)0xFF); 10974 } 10975 10976 void Assembler::fdecstp() { 10977 emit_int16((unsigned char)0xD9, (unsigned char)0xF6); 10978 } 10979 10980 void Assembler::fdiv(int i) { 10981 emit_farith(0xD8, 0xF0, i); 10982 } 10983 10984 void Assembler::fdiv_d(Address src) { 10985 InstructionMark im(this); 10986 emit_int8((unsigned char)0xDC); 10987 emit_operand32(rsi, src, 0); 10988 } 10989 10990 void Assembler::fdiv_s(Address src) { 10991 InstructionMark im(this); 10992 emit_int8((unsigned char)0xD8); 10993 emit_operand32(rsi, src, 0); 10994 } 10995 10996 void Assembler::fdiva(int i) { 10997 emit_farith(0xDC, 0xF8, i); 10998 } 10999 11000 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994) 11001 // is erroneous for some of the floating-point instructions below. 11002 11003 void Assembler::fdivp(int i) { 11004 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong) 11005 } 11006 11007 void Assembler::fdivr(int i) { 11008 emit_farith(0xD8, 0xF8, i); 11009 } 11010 11011 void Assembler::fdivr_d(Address src) { 11012 InstructionMark im(this); 11013 emit_int8((unsigned char)0xDC); 11014 emit_operand32(rdi, src, 0); 11015 } 11016 11017 void Assembler::fdivr_s(Address src) { 11018 InstructionMark im(this); 11019 emit_int8((unsigned char)0xD8); 11020 emit_operand32(rdi, src, 0); 11021 } 11022 11023 void Assembler::fdivra(int i) { 11024 emit_farith(0xDC, 0xF0, i); 11025 } 11026 11027 void Assembler::fdivrp(int i) { 11028 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong) 11029 } 11030 11031 void Assembler::ffree(int i) { 11032 emit_farith(0xDD, 0xC0, i); 11033 } 11034 11035 void Assembler::fild_d(Address adr) { 11036 InstructionMark im(this); 11037 emit_int8((unsigned char)0xDF); 11038 emit_operand32(rbp, adr, 0); 11039 } 11040 11041 void Assembler::fild_s(Address adr) { 11042 InstructionMark im(this); 11043 emit_int8((unsigned char)0xDB); 11044 emit_operand32(rax, adr, 0); 11045 } 11046 11047 void Assembler::fincstp() { 11048 emit_int16((unsigned char)0xD9, (unsigned char)0xF7); 11049 } 11050 11051 void Assembler::finit() { 11052 emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3); 11053 } 11054 11055 void Assembler::fist_s(Address adr) { 11056 InstructionMark im(this); 11057 emit_int8((unsigned char)0xDB); 11058 emit_operand32(rdx, adr, 0); 11059 } 11060 11061 void Assembler::fistp_d(Address adr) { 11062 InstructionMark im(this); 11063 emit_int8((unsigned char)0xDF); 11064 emit_operand32(rdi, adr, 0); 11065 } 11066 11067 void Assembler::fistp_s(Address adr) { 11068 InstructionMark im(this); 11069 emit_int8((unsigned char)0xDB); 11070 emit_operand32(rbx, adr, 0); 11071 } 11072 11073 void Assembler::fld1() { 11074 emit_int16((unsigned char)0xD9, (unsigned char)0xE8); 11075 } 11076 11077 void Assembler::fld_d(Address adr) { 11078 InstructionMark im(this); 11079 emit_int8((unsigned char)0xDD); 11080 emit_operand32(rax, adr, 0); 11081 } 11082 11083 void Assembler::fld_s(Address adr) { 11084 InstructionMark im(this); 11085 emit_int8((unsigned char)0xD9); 11086 emit_operand32(rax, adr, 0); 11087 } 11088 11089 11090 void Assembler::fld_s(int index) { 11091 emit_farith(0xD9, 0xC0, index); 11092 } 11093 11094 void Assembler::fldcw(Address src) { 11095 InstructionMark im(this); 11096 emit_int8((unsigned char)0xD9); 11097 emit_operand32(rbp, src, 0); 11098 } 11099 11100 void Assembler::fldenv(Address src) { 11101 InstructionMark im(this); 11102 emit_int8((unsigned char)0xD9); 11103 emit_operand32(rsp, src, 0); 11104 } 11105 11106 void Assembler::fldlg2() { 11107 emit_int16((unsigned char)0xD9, (unsigned char)0xEC); 11108 } 11109 11110 void Assembler::fldln2() { 11111 emit_int16((unsigned char)0xD9, (unsigned char)0xED); 11112 } 11113 11114 void Assembler::fldz() { 11115 emit_int16((unsigned char)0xD9, (unsigned char)0xEE); 11116 } 11117 11118 void Assembler::flog() { 11119 fldln2(); 11120 fxch(); 11121 fyl2x(); 11122 } 11123 11124 void Assembler::flog10() { 11125 fldlg2(); 11126 fxch(); 11127 fyl2x(); 11128 } 11129 11130 void Assembler::fmul(int i) { 11131 emit_farith(0xD8, 0xC8, i); 11132 } 11133 11134 void Assembler::fmul_d(Address src) { 11135 InstructionMark im(this); 11136 emit_int8((unsigned char)0xDC); 11137 emit_operand32(rcx, src, 0); 11138 } 11139 11140 void Assembler::fmul_s(Address src) { 11141 InstructionMark im(this); 11142 emit_int8((unsigned char)0xD8); 11143 emit_operand32(rcx, src, 0); 11144 } 11145 11146 void Assembler::fmula(int i) { 11147 emit_farith(0xDC, 0xC8, i); 11148 } 11149 11150 void Assembler::fmulp(int i) { 11151 emit_farith(0xDE, 0xC8, i); 11152 } 11153 11154 void Assembler::fnsave(Address dst) { 11155 InstructionMark im(this); 11156 emit_int8((unsigned char)0xDD); 11157 emit_operand32(rsi, dst, 0); 11158 } 11159 11160 void Assembler::fnstcw(Address src) { 11161 InstructionMark im(this); 11162 emit_int16((unsigned char)0x9B, (unsigned char)0xD9); 11163 emit_operand32(rdi, src, 0); 11164 } 11165 11166 void Assembler::fnstsw_ax() { 11167 emit_int16((unsigned char)0xDF, (unsigned char)0xE0); 11168 } 11169 11170 void Assembler::fprem() { 11171 emit_int16((unsigned char)0xD9, (unsigned char)0xF8); 11172 } 11173 11174 void Assembler::fprem1() { 11175 emit_int16((unsigned char)0xD9, (unsigned char)0xF5); 11176 } 11177 11178 void Assembler::frstor(Address src) { 11179 InstructionMark im(this); 11180 emit_int8((unsigned char)0xDD); 11181 emit_operand32(rsp, src, 0); 11182 } 11183 11184 void Assembler::fsin() { 11185 emit_int16((unsigned char)0xD9, (unsigned char)0xFE); 11186 } 11187 11188 void Assembler::fsqrt() { 11189 emit_int16((unsigned char)0xD9, (unsigned char)0xFA); 11190 } 11191 11192 void Assembler::fst_d(Address adr) { 11193 InstructionMark im(this); 11194 emit_int8((unsigned char)0xDD); 11195 emit_operand32(rdx, adr, 0); 11196 } 11197 11198 void Assembler::fst_s(Address adr) { 11199 InstructionMark im(this); 11200 emit_int8((unsigned char)0xD9); 11201 emit_operand32(rdx, adr, 0); 11202 } 11203 11204 void Assembler::fstp_d(Address adr) { 11205 InstructionMark im(this); 11206 emit_int8((unsigned char)0xDD); 11207 emit_operand32(rbx, adr, 0); 11208 } 11209 11210 void Assembler::fstp_d(int index) { 11211 emit_farith(0xDD, 0xD8, index); 11212 } 11213 11214 void Assembler::fstp_s(Address adr) { 11215 InstructionMark im(this); 11216 emit_int8((unsigned char)0xD9); 11217 emit_operand32(rbx, adr, 0); 11218 } 11219 11220 void Assembler::fsub(int i) { 11221 emit_farith(0xD8, 0xE0, i); 11222 } 11223 11224 void Assembler::fsub_d(Address src) { 11225 InstructionMark im(this); 11226 emit_int8((unsigned char)0xDC); 11227 emit_operand32(rsp, src, 0); 11228 } 11229 11230 void Assembler::fsub_s(Address src) { 11231 InstructionMark im(this); 11232 emit_int8((unsigned char)0xD8); 11233 emit_operand32(rsp, src, 0); 11234 } 11235 11236 void Assembler::fsuba(int i) { 11237 emit_farith(0xDC, 0xE8, i); 11238 } 11239 11240 void Assembler::fsubp(int i) { 11241 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong) 11242 } 11243 11244 void Assembler::fsubr(int i) { 11245 emit_farith(0xD8, 0xE8, i); 11246 } 11247 11248 void Assembler::fsubr_d(Address src) { 11249 InstructionMark im(this); 11250 emit_int8((unsigned char)0xDC); 11251 emit_operand32(rbp, src, 0); 11252 } 11253 11254 void Assembler::fsubr_s(Address src) { 11255 InstructionMark im(this); 11256 emit_int8((unsigned char)0xD8); 11257 emit_operand32(rbp, src, 0); 11258 } 11259 11260 void Assembler::fsubra(int i) { 11261 emit_farith(0xDC, 0xE0, i); 11262 } 11263 11264 void Assembler::fsubrp(int i) { 11265 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong) 11266 } 11267 11268 void Assembler::ftan() { 11269 emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8); 11270 } 11271 11272 void Assembler::ftst() { 11273 emit_int16((unsigned char)0xD9, (unsigned char)0xE4); 11274 } 11275 11276 void Assembler::fucomi(int i) { 11277 // make sure the instruction is supported (introduced for P6, together with cmov) 11278 guarantee(VM_Version::supports_cmov(), "illegal instruction"); 11279 emit_farith(0xDB, 0xE8, i); 11280 } 11281 11282 void Assembler::fucomip(int i) { 11283 // make sure the instruction is supported (introduced for P6, together with cmov) 11284 guarantee(VM_Version::supports_cmov(), "illegal instruction"); 11285 emit_farith(0xDF, 0xE8, i); 11286 } 11287 11288 void Assembler::fwait() { 11289 emit_int8((unsigned char)0x9B); 11290 } 11291 11292 void Assembler::fxch(int i) { 11293 emit_farith(0xD9, 0xC8, i); 11294 } 11295 11296 void Assembler::fyl2x() { 11297 emit_int16((unsigned char)0xD9, (unsigned char)0xF1); 11298 } 11299 11300 void Assembler::frndint() { 11301 emit_int16((unsigned char)0xD9, (unsigned char)0xFC); 11302 } 11303 11304 void Assembler::f2xm1() { 11305 emit_int16((unsigned char)0xD9, (unsigned char)0xF0); 11306 } 11307 11308 void Assembler::fldl2e() { 11309 emit_int16((unsigned char)0xD9, (unsigned char)0xEA); 11310 } 11311 #endif // !_LP64 11312 11313 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding. 11314 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 }; 11315 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding. 11316 static int simd_opc[4] = { 0, 0, 0x38, 0x3A }; 11317 11318 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding. 11319 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) { 11320 if (pre > 0) { 11321 emit_int8(simd_pre[pre]); 11322 } 11323 if (rex_w) { 11324 prefixq(adr, xreg); 11325 } else { 11326 prefix(adr, xreg); 11327 } 11328 if (opc > 0) { 11329 emit_int8(0x0F); 11330 int opc2 = simd_opc[opc]; 11331 if (opc2 > 0) { 11332 emit_int8(opc2); 11333 } 11334 } 11335 } 11336 11337 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) { 11338 if (pre > 0) { 11339 emit_int8(simd_pre[pre]); 11340 } 11341 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc); 11342 if (opc > 0) { 11343 emit_int8(0x0F); 11344 int opc2 = simd_opc[opc]; 11345 if (opc2 > 0) { 11346 emit_int8(opc2); 11347 } 11348 } 11349 return encode; 11350 } 11351 11352 11353 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) { 11354 int vector_len = _attributes->get_vector_len(); 11355 bool vex_w = _attributes->is_rex_vex_w(); 11356 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) { 11357 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0); 11358 byte1 = (~byte1) & 0xE0; 11359 byte1 |= opc; 11360 11361 int byte2 = ((~nds_enc) & 0xf) << 3; 11362 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre; 11363 11364 emit_int24((unsigned char)VEX_3bytes, byte1, byte2); 11365 } else { 11366 int byte1 = vex_r ? VEX_R : 0; 11367 byte1 = (~byte1) & 0x80; 11368 byte1 |= ((~nds_enc) & 0xf) << 3; 11369 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre; 11370 emit_int16((unsigned char)VEX_2bytes, byte1); 11371 } 11372 } 11373 11374 // This is a 4 byte encoding 11375 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){ 11376 // EVEX 0x62 prefix 11377 // byte1 = EVEX_4bytes; 11378 11379 bool vex_w = _attributes->is_rex_vex_w(); 11380 int evex_encoding = (vex_w ? VEX_W : 0); 11381 // EVEX.b is not currently used for broadcast of single element or data rounding modes 11382 _attributes->set_evex_encoding(evex_encoding); 11383 11384 // P0: byte 2, initialized to RXBR`00mm 11385 // instead of not'd 11386 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0); 11387 byte2 = (~byte2) & 0xF0; 11388 // confine opc opcode extensions in mm bits to lower two bits 11389 // of form {0F, 0F_38, 0F_3A} 11390 byte2 |= opc; 11391 11392 // P1: byte 3 as Wvvvv1pp 11393 int byte3 = ((~nds_enc) & 0xf) << 3; 11394 // p[10] is always 1 11395 byte3 |= EVEX_F; 11396 byte3 |= (vex_w & 1) << 7; 11397 // confine pre opcode extensions in pp bits to lower two bits 11398 // of form {66, F3, F2} 11399 byte3 |= pre; 11400 11401 // P2: byte 4 as zL'Lbv'aaa 11402 // kregs are implemented in the low 3 bits as aaa 11403 int byte4 = (_attributes->is_no_reg_mask()) ? 11404 0 : 11405 _attributes->get_embedded_opmask_register_specifier(); 11406 // EVEX.v` for extending EVEX.vvvv or VIDX 11407 byte4 |= (evex_v ? 0: EVEX_V); 11408 // third EXEC.b for broadcast actions 11409 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0); 11410 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024 11411 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5; 11412 // last is EVEX.z for zero/merge actions 11413 if (_attributes->is_no_reg_mask() == false && 11414 _attributes->get_embedded_opmask_register_specifier() != 0) { 11415 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0); 11416 } 11417 11418 emit_int32(EVEX_4bytes, byte2, byte3, byte4); 11419 } 11420 11421 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) { 11422 bool vex_r = (xreg_enc & 8) == 8; 11423 bool vex_b = adr.base_needs_rex(); 11424 bool vex_x; 11425 if (adr.isxmmindex()) { 11426 vex_x = adr.xmmindex_needs_rex(); 11427 } else { 11428 vex_x = adr.index_needs_rex(); 11429 } 11430 set_attributes(attributes); 11431 attributes->set_current_assembler(this); 11432 11433 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction 11434 // is allowed in legacy mode and has resources which will fit in it. 11435 // Pure EVEX instructions will have is_evex_instruction set in their definition. 11436 if (!attributes->is_legacy_mode()) { 11437 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) { 11438 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) { 11439 attributes->set_is_legacy_mode(); 11440 } 11441 } 11442 } 11443 11444 if (UseAVX > 2) { 11445 assert(((!attributes->uses_vl()) || 11446 (attributes->get_vector_len() == AVX_512bit) || 11447 (!_legacy_mode_vl) || 11448 (attributes->is_legacy_mode())),"XMM register should be 0-15"); 11449 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15"); 11450 } 11451 11452 clear_managed(); 11453 if (UseAVX > 2 && !attributes->is_legacy_mode()) 11454 { 11455 bool evex_r = (xreg_enc >= 16); 11456 bool evex_v; 11457 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31) 11458 if (adr.isxmmindex()) { 11459 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false); 11460 } else { 11461 evex_v = (nds_enc >= 16); 11462 } 11463 attributes->set_is_evex_instruction(); 11464 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc); 11465 } else { 11466 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) { 11467 attributes->set_rex_vex_w(false); 11468 } 11469 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc); 11470 } 11471 } 11472 11473 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) { 11474 bool vex_r = (dst_enc & 8) == 8; 11475 bool vex_b = (src_enc & 8) == 8; 11476 bool vex_x = false; 11477 set_attributes(attributes); 11478 attributes->set_current_assembler(this); 11479 11480 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction 11481 // is allowed in legacy mode and has resources which will fit in it. 11482 // Pure EVEX instructions will have is_evex_instruction set in their definition. 11483 if (!attributes->is_legacy_mode()) { 11484 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) { 11485 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) && 11486 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) { 11487 attributes->set_is_legacy_mode(); 11488 } 11489 } 11490 } 11491 11492 if (UseAVX > 2) { 11493 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false 11494 // Instruction with uses_vl true are vector instructions 11495 // All the vector instructions with AVX_512bit length can have legacy_mode as false 11496 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported 11497 // Rest all should have legacy_mode set as true 11498 assert(((!attributes->uses_vl()) || 11499 (attributes->get_vector_len() == AVX_512bit) || 11500 (!_legacy_mode_vl) || 11501 (attributes->is_legacy_mode())),"XMM register should be 0-15"); 11502 // Instruction with legacy_mode true should have dst, nds and src < 15 11503 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15"); 11504 } 11505 11506 clear_managed(); 11507 if (UseAVX > 2 && !attributes->is_legacy_mode()) 11508 { 11509 bool evex_r = (dst_enc >= 16); 11510 bool evex_v = (nds_enc >= 16); 11511 // can use vex_x as bank extender on rm encoding 11512 vex_x = (src_enc >= 16); 11513 attributes->set_is_evex_instruction(); 11514 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc); 11515 } else { 11516 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) { 11517 attributes->set_rex_vex_w(false); 11518 } 11519 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc); 11520 } 11521 11522 // return modrm byte components for operands 11523 return (((dst_enc & 7) << 3) | (src_enc & 7)); 11524 } 11525 11526 11527 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, 11528 VexOpcode opc, InstructionAttr *attributes) { 11529 if (UseAVX > 0) { 11530 int xreg_enc = xreg->encoding(); 11531 int nds_enc = nds->is_valid() ? nds->encoding() : 0; 11532 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes); 11533 } else { 11534 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding"); 11535 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w()); 11536 } 11537 } 11538 11539 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, 11540 VexOpcode opc, InstructionAttr *attributes) { 11541 int dst_enc = dst->encoding(); 11542 int src_enc = src->encoding(); 11543 if (UseAVX > 0) { 11544 int nds_enc = nds->is_valid() ? nds->encoding() : 0; 11545 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes); 11546 } else { 11547 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding"); 11548 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w()); 11549 } 11550 } 11551 11552 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 11553 assert(VM_Version::supports_avx(), ""); 11554 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 11555 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 11556 emit_int16(0x5F, (0xC0 | encode)); 11557 } 11558 11559 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 11560 assert(VM_Version::supports_avx(), ""); 11561 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 11562 attributes.set_rex_vex_w_reverted(); 11563 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 11564 emit_int16(0x5F, (0xC0 | encode)); 11565 } 11566 11567 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) { 11568 assert(VM_Version::supports_avx(), ""); 11569 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 11570 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 11571 emit_int16(0x5D, (0xC0 | encode)); 11572 } 11573 11574 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { 11575 assert(VM_Version::supports_avx(), ""); 11576 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 11577 attributes.set_rex_vex_w_reverted(); 11578 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 11579 emit_int16(0x5D, (0xC0 | encode)); 11580 } 11581 11582 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) { 11583 assert(VM_Version::supports_avx(), ""); 11584 assert(vector_len <= AVX_256bit, ""); 11585 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 11586 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11587 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop)); 11588 } 11589 11590 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 11591 assert(VM_Version::supports_avx(), ""); 11592 assert(vector_len <= AVX_256bit, ""); 11593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 11594 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11595 int src2_enc = src2->encoding(); 11596 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4)); 11597 } 11598 11599 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 11600 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), ""); 11601 assert(vector_len <= AVX_256bit, ""); 11602 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 11603 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11604 int src2_enc = src2->encoding(); 11605 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4)); 11606 } 11607 11608 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 11609 assert(VM_Version::supports_avx2(), ""); 11610 assert(vector_len <= AVX_256bit, ""); 11611 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 11612 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11613 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8); 11614 } 11615 11616 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) { 11617 assert(VM_Version::supports_avx(), ""); 11618 assert(vector_len <= AVX_256bit, ""); 11619 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 11620 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11621 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison); 11622 } 11623 11624 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11625 ComparisonPredicateFP comparison, int vector_len) { 11626 assert(VM_Version::supports_evex(), ""); 11627 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib 11628 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11629 attributes.set_is_evex_instruction(); 11630 attributes.set_embedded_opmask_register_specifier(mask); 11631 attributes.reset_is_clear_context(); 11632 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 11633 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 11634 } 11635 11636 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11637 ComparisonPredicateFP comparison, int vector_len) { 11638 assert(VM_Version::supports_evex(), ""); 11639 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib 11640 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11641 attributes.set_is_evex_instruction(); 11642 attributes.set_embedded_opmask_register_specifier(mask); 11643 attributes.reset_is_clear_context(); 11644 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11645 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison); 11646 } 11647 11648 void Assembler::blendvps(XMMRegister dst, XMMRegister src) { 11649 assert(VM_Version::supports_sse4_1(), ""); 11650 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 11651 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11652 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11653 emit_int16(0x14, (0xC0 | encode)); 11654 } 11655 11656 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) { 11657 assert(VM_Version::supports_sse4_1(), ""); 11658 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 11659 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11660 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11661 emit_int16(0x15, (0xC0 | encode)); 11662 } 11663 11664 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) { 11665 assert(VM_Version::supports_sse4_1(), ""); 11666 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding"); 11667 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11668 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11669 emit_int16(0x10, (0xC0 | encode)); 11670 } 11671 11672 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) { 11673 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), ""); 11674 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11675 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11676 int src2_enc = src2->encoding(); 11677 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4)); 11678 } 11679 11680 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) { 11681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11683 emit_int24(0x0C, (0xC0 | encode), imm8); 11684 } 11685 11686 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 11687 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 11688 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 11689 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11690 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11691 emit_int16(0x64, (0xC0 | encode)); 11692 } 11693 11694 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 11695 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 11696 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 11697 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11698 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11699 emit_int16(0x65, (0xC0 | encode)); 11700 } 11701 11702 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 11703 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 11704 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 11705 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11706 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11707 emit_int16(0x66, (0xC0 | encode)); 11708 } 11709 11710 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 11711 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), ""); 11712 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest"); 11713 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11715 emit_int16(0x37, (0xC0 | encode)); 11716 } 11717 11718 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11719 int comparison, bool is_signed, int vector_len) { 11720 assert(VM_Version::supports_evex(), ""); 11721 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11722 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib 11723 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11724 attributes.set_is_evex_instruction(); 11725 attributes.set_embedded_opmask_register_specifier(mask); 11726 attributes.reset_is_clear_context(); 11727 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11728 int opcode = is_signed ? 0x1F : 0x1E; 11729 emit_int24(opcode, (0xC0 | encode), comparison); 11730 } 11731 11732 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 11733 int comparison, bool is_signed, int vector_len) { 11734 assert(VM_Version::supports_evex(), ""); 11735 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11736 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib 11737 InstructionMark im(this); 11738 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11739 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 11740 attributes.set_is_evex_instruction(); 11741 attributes.set_embedded_opmask_register_specifier(mask); 11742 attributes.reset_is_clear_context(); 11743 int dst_enc = kdst->encoding(); 11744 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11745 int opcode = is_signed ? 0x1F : 0x1E; 11746 emit_int8((unsigned char)opcode); 11747 emit_operand(as_Register(dst_enc), src, 1); 11748 emit_int8((unsigned char)comparison); 11749 } 11750 11751 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11752 int comparison, bool is_signed, int vector_len) { 11753 assert(VM_Version::supports_evex(), ""); 11754 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11755 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib 11756 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11757 attributes.set_is_evex_instruction(); 11758 attributes.set_embedded_opmask_register_specifier(mask); 11759 attributes.reset_is_clear_context(); 11760 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11761 int opcode = is_signed ? 0x1F : 0x1E; 11762 emit_int24(opcode, (0xC0 | encode), comparison); 11763 } 11764 11765 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 11766 int comparison, bool is_signed, int vector_len) { 11767 assert(VM_Version::supports_evex(), ""); 11768 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11769 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib 11770 InstructionMark im(this); 11771 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11772 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit); 11773 attributes.set_is_evex_instruction(); 11774 attributes.set_embedded_opmask_register_specifier(mask); 11775 attributes.reset_is_clear_context(); 11776 int dst_enc = kdst->encoding(); 11777 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11778 int opcode = is_signed ? 0x1F : 0x1E; 11779 emit_int8((unsigned char)opcode); 11780 emit_operand(as_Register(dst_enc), src, 1); 11781 emit_int8((unsigned char)comparison); 11782 } 11783 11784 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11785 int comparison, bool is_signed, int vector_len) { 11786 assert(VM_Version::supports_evex(), ""); 11787 assert(VM_Version::supports_avx512bw(), ""); 11788 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11789 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib 11790 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 11791 attributes.set_is_evex_instruction(); 11792 attributes.set_embedded_opmask_register_specifier(mask); 11793 attributes.reset_is_clear_context(); 11794 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11795 int opcode = is_signed ? 0x3F : 0x3E; 11796 emit_int24(opcode, (0xC0 | encode), comparison); 11797 } 11798 11799 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 11800 int comparison, bool is_signed, int vector_len) { 11801 assert(VM_Version::supports_evex(), ""); 11802 assert(VM_Version::supports_avx512bw(), ""); 11803 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11804 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib 11805 InstructionMark im(this); 11806 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 11807 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 11808 attributes.set_is_evex_instruction(); 11809 attributes.set_embedded_opmask_register_specifier(mask); 11810 attributes.reset_is_clear_context(); 11811 int dst_enc = kdst->encoding(); 11812 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11813 int opcode = is_signed ? 0x3F : 0x3E; 11814 emit_int8((unsigned char)opcode); 11815 emit_operand(as_Register(dst_enc), src, 1); 11816 emit_int8((unsigned char)comparison); 11817 } 11818 11819 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, 11820 int comparison, bool is_signed, int vector_len) { 11821 assert(VM_Version::supports_evex(), ""); 11822 assert(VM_Version::supports_avx512bw(), ""); 11823 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11824 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib 11825 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 11826 attributes.set_is_evex_instruction(); 11827 attributes.set_embedded_opmask_register_specifier(mask); 11828 attributes.reset_is_clear_context(); 11829 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11830 int opcode = is_signed ? 0x3F : 0x3E; 11831 emit_int24(opcode, (0xC0 | encode), comparison); 11832 } 11833 11834 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src, 11835 int comparison, bool is_signed, int vector_len) { 11836 assert(VM_Version::supports_evex(), ""); 11837 assert(VM_Version::supports_avx512bw(), ""); 11838 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, ""); 11839 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib 11840 InstructionMark im(this); 11841 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 11842 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit); 11843 attributes.set_is_evex_instruction(); 11844 attributes.set_embedded_opmask_register_specifier(mask); 11845 attributes.reset_is_clear_context(); 11846 int dst_enc = kdst->encoding(); 11847 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11848 int opcode = is_signed ? 0x3F : 0x3E; 11849 emit_int8((unsigned char)opcode); 11850 emit_operand(as_Register(dst_enc), src, 1); 11851 emit_int8((unsigned char)comparison); 11852 } 11853 11854 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11855 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11857 attributes.set_is_evex_instruction(); 11858 attributes.set_embedded_opmask_register_specifier(mask); 11859 if (merge) { 11860 attributes.reset_is_clear_context(); 11861 } 11862 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11863 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11864 } 11865 11866 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11867 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11868 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11869 attributes.set_is_evex_instruction(); 11870 attributes.set_embedded_opmask_register_specifier(mask); 11871 if (merge) { 11872 attributes.reset_is_clear_context(); 11873 } 11874 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11875 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11876 } 11877 11878 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11879 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11880 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11881 attributes.set_is_evex_instruction(); 11882 attributes.set_embedded_opmask_register_specifier(mask); 11883 if (merge) { 11884 attributes.reset_is_clear_context(); 11885 } 11886 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11887 emit_int16(0x14, (0xC0 | encode)); 11888 } 11889 11890 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11891 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11892 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11893 attributes.set_is_evex_instruction(); 11894 attributes.set_embedded_opmask_register_specifier(mask); 11895 if (merge) { 11896 attributes.reset_is_clear_context(); 11897 } 11898 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11899 emit_int16(0x14, (0xC0 | encode)); 11900 } 11901 11902 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11903 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11904 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11905 attributes.set_is_evex_instruction(); 11906 attributes.set_embedded_opmask_register_specifier(mask); 11907 if (merge) { 11908 attributes.reset_is_clear_context(); 11909 } 11910 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11911 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11912 } 11913 11914 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) { 11915 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11916 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11917 attributes.set_is_evex_instruction(); 11918 attributes.set_embedded_opmask_register_specifier(mask); 11919 if (merge) { 11920 attributes.reset_is_clear_context(); 11921 } 11922 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 11923 emit_int24(0x72, (0xC0 | encode), shift & 0xFF); 11924 } 11925 11926 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11927 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11928 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11929 attributes.set_is_evex_instruction(); 11930 attributes.set_embedded_opmask_register_specifier(mask); 11931 if (merge) { 11932 attributes.reset_is_clear_context(); 11933 } 11934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11935 emit_int16(0x15, (0xC0 | encode)); 11936 } 11937 11938 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11939 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 11940 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11941 attributes.set_is_evex_instruction(); 11942 attributes.set_embedded_opmask_register_specifier(mask); 11943 if (merge) { 11944 attributes.reset_is_clear_context(); 11945 } 11946 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11947 emit_int16(0x15, (0xC0 | encode)); 11948 } 11949 11950 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) { 11951 assert(VM_Version::supports_avx(), ""); 11952 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 11953 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); 11954 int mask_enc = mask->encoding(); 11955 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4); 11956 } 11957 11958 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11959 assert(VM_Version::supports_evex(), ""); 11960 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r 11961 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11962 attributes.set_is_evex_instruction(); 11963 attributes.set_embedded_opmask_register_specifier(mask); 11964 if (merge) { 11965 attributes.reset_is_clear_context(); 11966 } 11967 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11968 emit_int16(0x65, (0xC0 | encode)); 11969 } 11970 11971 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11972 assert(VM_Version::supports_evex(), ""); 11973 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r 11974 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 11975 attributes.set_is_evex_instruction(); 11976 attributes.set_embedded_opmask_register_specifier(mask); 11977 if (merge) { 11978 attributes.reset_is_clear_context(); 11979 } 11980 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11981 emit_int16(0x65, (0xC0 | encode)); 11982 } 11983 11984 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11985 assert(VM_Version::supports_evex(), ""); 11986 assert(VM_Version::supports_avx512bw(), ""); 11987 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r 11988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 11989 attributes.set_is_evex_instruction(); 11990 attributes.set_embedded_opmask_register_specifier(mask); 11991 if (merge) { 11992 attributes.reset_is_clear_context(); 11993 } 11994 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 11995 emit_int16(0x66, (0xC0 | encode)); 11996 } 11997 11998 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 11999 assert(VM_Version::supports_evex(), ""); 12000 assert(VM_Version::supports_avx512bw(), ""); 12001 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r 12002 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true); 12003 attributes.set_is_evex_instruction(); 12004 attributes.set_embedded_opmask_register_specifier(mask); 12005 if (merge) { 12006 attributes.reset_is_clear_context(); 12007 } 12008 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12009 emit_int16(0x66, (0xC0 | encode)); 12010 } 12011 12012 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12013 assert(VM_Version::supports_evex(), ""); 12014 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r 12015 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12016 attributes.set_is_evex_instruction(); 12017 attributes.set_embedded_opmask_register_specifier(mask); 12018 if (merge) { 12019 attributes.reset_is_clear_context(); 12020 } 12021 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12022 emit_int16(0x64, (0xC0 | encode)); 12023 } 12024 12025 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 12026 assert(VM_Version::supports_evex(), ""); 12027 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r 12028 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12029 attributes.set_is_evex_instruction(); 12030 attributes.set_embedded_opmask_register_specifier(mask); 12031 if (merge) { 12032 attributes.reset_is_clear_context(); 12033 } 12034 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12035 emit_int16(0x64, (0xC0 | encode)); 12036 } 12037 12038 void Assembler::bzhiq(Register dst, Register src1, Register src2) { 12039 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12040 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12041 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12042 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 12043 } 12044 12045 void Assembler::pextl(Register dst, Register src1, Register src2) { 12046 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12047 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12048 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12049 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 12050 } 12051 12052 void Assembler::pdepl(Register dst, Register src1, Register src2) { 12053 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12054 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12055 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12056 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 12057 } 12058 12059 void Assembler::pextq(Register dst, Register src1, Register src2) { 12060 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12061 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12062 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12063 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 12064 } 12065 12066 void Assembler::pdepq(Register dst, Register src1, Register src2) { 12067 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12068 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12069 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12070 emit_int16((unsigned char)0xF5, (0xC0 | encode)); 12071 } 12072 12073 void Assembler::pextl(Register dst, Register src1, Address src2) { 12074 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12075 InstructionMark im(this); 12076 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12077 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12078 emit_int8((unsigned char)0xF5); 12079 emit_operand(dst, src2, 0); 12080 } 12081 12082 void Assembler::pdepl(Register dst, Register src1, Address src2) { 12083 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12084 InstructionMark im(this); 12085 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12086 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12087 emit_int8((unsigned char)0xF5); 12088 emit_operand(dst, src2, 0); 12089 } 12090 12091 void Assembler::pextq(Register dst, Register src1, Address src2) { 12092 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12093 InstructionMark im(this); 12094 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12095 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12096 emit_int8((unsigned char)0xF5); 12097 emit_operand(dst, src2, 0); 12098 } 12099 12100 void Assembler::pdepq(Register dst, Register src1, Address src2) { 12101 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 12102 InstructionMark im(this); 12103 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12104 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12105 emit_int8((unsigned char)0xF5); 12106 emit_operand(dst, src2, 0); 12107 } 12108 12109 void Assembler::sarxl(Register dst, Register src1, Register src2) { 12110 assert(VM_Version::supports_bmi2(), ""); 12111 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12112 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12113 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12114 } 12115 12116 void Assembler::sarxl(Register dst, Address src1, Register src2) { 12117 assert(VM_Version::supports_bmi2(), ""); 12118 InstructionMark im(this); 12119 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12120 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12121 emit_int8((unsigned char)0xF7); 12122 emit_operand(dst, src1, 0); 12123 } 12124 12125 void Assembler::sarxq(Register dst, Register src1, Register src2) { 12126 assert(VM_Version::supports_bmi2(), ""); 12127 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12128 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12129 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12130 } 12131 12132 void Assembler::sarxq(Register dst, Address src1, Register src2) { 12133 assert(VM_Version::supports_bmi2(), ""); 12134 InstructionMark im(this); 12135 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12136 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12137 emit_int8((unsigned char)0xF7); 12138 emit_operand(dst, src1, 0); 12139 } 12140 12141 void Assembler::shlxl(Register dst, Register src1, Register src2) { 12142 assert(VM_Version::supports_bmi2(), ""); 12143 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12144 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12145 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12146 } 12147 12148 void Assembler::shlxl(Register dst, Address src1, Register src2) { 12149 assert(VM_Version::supports_bmi2(), ""); 12150 InstructionMark im(this); 12151 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12152 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12153 emit_int8((unsigned char)0xF7); 12154 emit_operand(dst, src1, 0); 12155 } 12156 12157 void Assembler::shlxq(Register dst, Register src1, Register src2) { 12158 assert(VM_Version::supports_bmi2(), ""); 12159 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12160 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12161 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12162 } 12163 12164 void Assembler::shlxq(Register dst, Address src1, Register src2) { 12165 assert(VM_Version::supports_bmi2(), ""); 12166 InstructionMark im(this); 12167 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12168 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12169 emit_int8((unsigned char)0xF7); 12170 emit_operand(dst, src1, 0); 12171 } 12172 12173 void Assembler::shrxl(Register dst, Register src1, Register src2) { 12174 assert(VM_Version::supports_bmi2(), ""); 12175 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12176 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12177 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12178 } 12179 12180 void Assembler::shrxl(Register dst, Address src1, Register src2) { 12181 assert(VM_Version::supports_bmi2(), ""); 12182 InstructionMark im(this); 12183 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12184 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12185 emit_int8((unsigned char)0xF7); 12186 emit_operand(dst, src1, 0); 12187 } 12188 12189 void Assembler::shrxq(Register dst, Register src1, Register src2) { 12190 assert(VM_Version::supports_bmi2(), ""); 12191 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12192 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12193 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 12194 } 12195 12196 void Assembler::shrxq(Register dst, Address src1, Register src2) { 12197 assert(VM_Version::supports_bmi2(), ""); 12198 InstructionMark im(this); 12199 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true); 12200 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 12201 emit_int8((unsigned char)0xF7); 12202 emit_operand(dst, src1, 0); 12203 } 12204 12205 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) { 12206 assert(VM_Version::supports_avx512vldq(), ""); 12207 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12208 attributes.set_is_evex_instruction(); 12209 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12210 emit_int16(0x39, (0xC0 | encode)); 12211 } 12212 12213 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) { 12214 assert(VM_Version::supports_avx512vldq(), ""); 12215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12216 attributes.set_is_evex_instruction(); 12217 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12218 emit_int16(0x39, (0xC0 | encode)); 12219 } 12220 12221 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) { 12222 assert(VM_Version::supports_avx512vlbw(), ""); 12223 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12224 attributes.set_is_evex_instruction(); 12225 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12226 emit_int16(0x29, (0xC0 | encode)); 12227 } 12228 12229 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) { 12230 assert(VM_Version::supports_avx512vlbw(), ""); 12231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12232 attributes.set_is_evex_instruction(); 12233 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12234 emit_int16(0x29, (0xC0 | encode)); 12235 } 12236 12237 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) { 12238 assert(VM_Version::supports_avx512vldq(), ""); 12239 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12240 attributes.set_is_evex_instruction(); 12241 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12242 emit_int16(0x38, (0xC0 | encode)); 12243 } 12244 12245 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) { 12246 assert(VM_Version::supports_avx512vldq(), ""); 12247 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12248 attributes.set_is_evex_instruction(); 12249 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12250 emit_int16(0x38, (0xC0 | encode)); 12251 } 12252 12253 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) { 12254 assert(VM_Version::supports_avx512vlbw(), ""); 12255 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12256 attributes.set_is_evex_instruction(); 12257 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12258 emit_int16(0x28, (0xC0 | encode)); 12259 } 12260 12261 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) { 12262 assert(VM_Version::supports_avx512vlbw(), ""); 12263 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); 12264 attributes.set_is_evex_instruction(); 12265 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); 12266 emit_int16(0x28, (0xC0 | encode)); 12267 } 12268 12269 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12270 assert(VM_Version::supports_avx512_vbmi2(), ""); 12271 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12272 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12273 attributes.set_embedded_opmask_register_specifier(mask); 12274 attributes.set_is_evex_instruction(); 12275 if (merge) { 12276 attributes.reset_is_clear_context(); 12277 } 12278 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12279 emit_int16((unsigned char)0x63, (0xC0 | encode)); 12280 } 12281 12282 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12283 assert(VM_Version::supports_avx512_vbmi2(), ""); 12284 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12285 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12286 attributes.set_embedded_opmask_register_specifier(mask); 12287 attributes.set_is_evex_instruction(); 12288 if (merge) { 12289 attributes.reset_is_clear_context(); 12290 } 12291 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12292 emit_int16((unsigned char)0x63, (0xC0 | encode)); 12293 } 12294 12295 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12296 assert(VM_Version::supports_evex(), ""); 12297 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12299 attributes.set_embedded_opmask_register_specifier(mask); 12300 attributes.set_is_evex_instruction(); 12301 if (merge) { 12302 attributes.reset_is_clear_context(); 12303 } 12304 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12305 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 12306 } 12307 12308 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12309 assert(VM_Version::supports_evex(), ""); 12310 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12311 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12312 attributes.set_embedded_opmask_register_specifier(mask); 12313 attributes.set_is_evex_instruction(); 12314 if (merge) { 12315 attributes.reset_is_clear_context(); 12316 } 12317 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12318 emit_int16((unsigned char)0x8B, (0xC0 | encode)); 12319 } 12320 12321 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12322 assert(VM_Version::supports_evex(), ""); 12323 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12324 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12325 attributes.set_embedded_opmask_register_specifier(mask); 12326 attributes.set_is_evex_instruction(); 12327 if (merge) { 12328 attributes.reset_is_clear_context(); 12329 } 12330 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12331 emit_int16((unsigned char)0x8A, (0xC0 | encode)); 12332 } 12333 12334 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { 12335 assert(VM_Version::supports_evex(), ""); 12336 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); 12337 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true); 12338 attributes.set_embedded_opmask_register_specifier(mask); 12339 attributes.set_is_evex_instruction(); 12340 if (merge) { 12341 attributes.reset_is_clear_context(); 12342 } 12343 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); 12344 emit_int16((unsigned char)0x8A, (0xC0 | encode)); 12345 } 12346 12347 #ifndef _LP64 12348 12349 void Assembler::incl(Register dst) { 12350 // Don't use it directly. Use MacroAssembler::incrementl() instead. 12351 emit_int8(0x40 | dst->encoding()); 12352 } 12353 12354 void Assembler::lea(Register dst, Address src) { 12355 leal(dst, src); 12356 } 12357 12358 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) { 12359 InstructionMark im(this); 12360 emit_int8((unsigned char)0xC7); 12361 emit_operand(rax, dst, 4); 12362 emit_data((int)imm32, rspec, 0); 12363 } 12364 12365 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) { 12366 InstructionMark im(this); 12367 int encode = prefix_and_encode(dst->encoding()); 12368 emit_int8((0xB8 | encode)); 12369 emit_data((int)imm32, rspec, 0); 12370 } 12371 12372 void Assembler::popa() { // 32bit 12373 emit_int8(0x61); 12374 } 12375 12376 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) { 12377 InstructionMark im(this); 12378 emit_int8(0x68); 12379 emit_data(imm32, rspec, 0); 12380 } 12381 12382 void Assembler::pusha() { // 32bit 12383 emit_int8(0x60); 12384 } 12385 12386 void Assembler::set_byte_if_not_zero(Register dst) { 12387 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding())); 12388 } 12389 12390 #else // LP64 12391 12392 // 64bit only pieces of the assembler 12393 12394 void Assembler::set_byte_if_not_zero(Register dst) { 12395 int enc = prefix_and_encode(dst->encoding(), true); 12396 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc)); 12397 } 12398 12399 // This should only be used by 64bit instructions that can use rip-relative 12400 // it cannot be used by instructions that want an immediate value. 12401 12402 // Determine whether an address is always reachable in rip-relative addressing mode 12403 // when accessed from the code cache. 12404 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) { 12405 switch (reloc_type) { 12406 // This should be rip-relative and easily reachable. 12407 case relocInfo::internal_word_type: { 12408 return true; 12409 } 12410 // This should be rip-relative within the code cache and easily 12411 // reachable until we get huge code caches. (At which point 12412 // IC code is going to have issues). 12413 case relocInfo::virtual_call_type: 12414 case relocInfo::opt_virtual_call_type: 12415 case relocInfo::static_call_type: 12416 case relocInfo::static_stub_type: { 12417 return true; 12418 } 12419 case relocInfo::runtime_call_type: 12420 case relocInfo::external_word_type: 12421 case relocInfo::poll_return_type: // these are really external_word but need special 12422 case relocInfo::poll_type: { // relocs to identify them 12423 return CodeCache::contains(target); 12424 } 12425 default: { 12426 return false; 12427 } 12428 } 12429 } 12430 12431 // Determine whether an address is reachable in rip-relative addressing mode from the code cache. 12432 static bool is_reachable(address target, relocInfo::relocType reloc_type) { 12433 if (is_always_reachable(target, reloc_type)) { 12434 return true; 12435 } 12436 switch (reloc_type) { 12437 // None will force a 64bit literal to the code stream. Likely a placeholder 12438 // for something that will be patched later and we need to certain it will 12439 // always be reachable. 12440 case relocInfo::none: { 12441 return false; 12442 } 12443 case relocInfo::runtime_call_type: 12444 case relocInfo::external_word_type: 12445 case relocInfo::poll_return_type: // these are really external_word but need special 12446 case relocInfo::poll_type: { // relocs to identify them 12447 assert(!CodeCache::contains(target), "always reachable"); 12448 if (ForceUnreachable) { 12449 return false; // stress the correction code 12450 } 12451 // For external_word_type/runtime_call_type if it is reachable from where we 12452 // are now (possibly a temp buffer) and where we might end up 12453 // anywhere in the code cache then we are always reachable. 12454 // This would have to change if we ever save/restore shared code to be more pessimistic. 12455 // Code buffer has to be allocated in the code cache, so check against 12456 // code cache boundaries cover that case. 12457 // 12458 // In rip-relative addressing mode, an effective address is formed by adding displacement 12459 // to the 64-bit RIP of the next instruction which is not known yet. Considering target address 12460 // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough 12461 // to account for that. 12462 return Assembler::is_simm32(target - CodeCache::low_bound()) && 12463 Assembler::is_simm32(target - CodeCache::high_bound()); 12464 } 12465 default: { 12466 return false; 12467 } 12468 } 12469 } 12470 12471 bool Assembler::reachable(AddressLiteral adr) { 12472 assert(CodeCache::contains(pc()), "required"); 12473 if (adr.is_lval()) { 12474 return false; 12475 } 12476 return is_reachable(adr.target(), adr.reloc()); 12477 } 12478 12479 bool Assembler::always_reachable(AddressLiteral adr) { 12480 assert(CodeCache::contains(pc()), "required"); 12481 if (adr.is_lval()) { 12482 return false; 12483 } 12484 return is_always_reachable(adr.target(), adr.reloc()); 12485 } 12486 12487 void Assembler::emit_data64(jlong data, 12488 relocInfo::relocType rtype, 12489 int format) { 12490 if (rtype == relocInfo::none) { 12491 emit_int64(data); 12492 } else { 12493 emit_data64(data, Relocation::spec_simple(rtype), format); 12494 } 12495 } 12496 12497 void Assembler::emit_data64(jlong data, 12498 RelocationHolder const& rspec, 12499 int format) { 12500 assert(imm_operand == 0, "default format must be immediate in this file"); 12501 assert(imm_operand == format, "must be immediate"); 12502 assert(inst_mark() != NULL, "must be inside InstructionMark"); 12503 // Do not use AbstractAssembler::relocate, which is not intended for 12504 // embedded words. Instead, relocate to the enclosing instruction. 12505 code_section()->relocate(inst_mark(), rspec, format); 12506 #ifdef ASSERT 12507 check_relocation(rspec, format); 12508 #endif 12509 emit_int64(data); 12510 } 12511 12512 void Assembler::prefix(Register reg) { 12513 if (reg->encoding() >= 8) { 12514 prefix(REX_B); 12515 } 12516 } 12517 12518 void Assembler::prefix(Register dst, Register src, Prefix p) { 12519 if (src->encoding() >= 8) { 12520 p = (Prefix)(p | REX_B); 12521 } 12522 if (dst->encoding() >= 8) { 12523 p = (Prefix)(p | REX_R); 12524 } 12525 if (p != Prefix_EMPTY) { 12526 // do not generate an empty prefix 12527 prefix(p); 12528 } 12529 } 12530 12531 void Assembler::prefix(Register dst, Address adr, Prefix p) { 12532 if (adr.base_needs_rex()) { 12533 if (adr.index_needs_rex()) { 12534 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X"); 12535 } else { 12536 prefix(REX_B); 12537 } 12538 } else { 12539 if (adr.index_needs_rex()) { 12540 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X"); 12541 } 12542 } 12543 if (dst->encoding() >= 8) { 12544 p = (Prefix)(p | REX_R); 12545 } 12546 if (p != Prefix_EMPTY) { 12547 // do not generate an empty prefix 12548 prefix(p); 12549 } 12550 } 12551 12552 void Assembler::prefix(Address adr) { 12553 if (adr.base_needs_rex()) { 12554 if (adr.index_needs_rex()) { 12555 prefix(REX_XB); 12556 } else { 12557 prefix(REX_B); 12558 } 12559 } else { 12560 if (adr.index_needs_rex()) { 12561 prefix(REX_X); 12562 } 12563 } 12564 } 12565 12566 void Assembler::prefix(Address adr, Register reg, bool byteinst) { 12567 if (reg->encoding() < 8) { 12568 if (adr.base_needs_rex()) { 12569 if (adr.index_needs_rex()) { 12570 prefix(REX_XB); 12571 } else { 12572 prefix(REX_B); 12573 } 12574 } else { 12575 if (adr.index_needs_rex()) { 12576 prefix(REX_X); 12577 } else if (byteinst && reg->encoding() >= 4) { 12578 prefix(REX); 12579 } 12580 } 12581 } else { 12582 if (adr.base_needs_rex()) { 12583 if (adr.index_needs_rex()) { 12584 prefix(REX_RXB); 12585 } else { 12586 prefix(REX_RB); 12587 } 12588 } else { 12589 if (adr.index_needs_rex()) { 12590 prefix(REX_RX); 12591 } else { 12592 prefix(REX_R); 12593 } 12594 } 12595 } 12596 } 12597 12598 void Assembler::prefix(Address adr, XMMRegister reg) { 12599 if (reg->encoding() < 8) { 12600 if (adr.base_needs_rex()) { 12601 if (adr.index_needs_rex()) { 12602 prefix(REX_XB); 12603 } else { 12604 prefix(REX_B); 12605 } 12606 } else { 12607 if (adr.index_needs_rex()) { 12608 prefix(REX_X); 12609 } 12610 } 12611 } else { 12612 if (adr.base_needs_rex()) { 12613 if (adr.index_needs_rex()) { 12614 prefix(REX_RXB); 12615 } else { 12616 prefix(REX_RB); 12617 } 12618 } else { 12619 if (adr.index_needs_rex()) { 12620 prefix(REX_RX); 12621 } else { 12622 prefix(REX_R); 12623 } 12624 } 12625 } 12626 } 12627 12628 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) { 12629 if (reg_enc >= 8) { 12630 prefix(REX_B); 12631 reg_enc -= 8; 12632 } else if (byteinst && reg_enc >= 4) { 12633 prefix(REX); 12634 } 12635 return reg_enc; 12636 } 12637 12638 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) { 12639 if (dst_enc < 8) { 12640 if (src_enc >= 8) { 12641 prefix(REX_B); 12642 src_enc -= 8; 12643 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) { 12644 prefix(REX); 12645 } 12646 } else { 12647 if (src_enc < 8) { 12648 prefix(REX_R); 12649 } else { 12650 prefix(REX_RB); 12651 src_enc -= 8; 12652 } 12653 dst_enc -= 8; 12654 } 12655 return dst_enc << 3 | src_enc; 12656 } 12657 12658 int8_t Assembler::get_prefixq(Address adr) { 12659 int8_t prfx = get_prefixq(adr, rax); 12660 assert(REX_W <= prfx && prfx <= REX_WXB, "must be"); 12661 return prfx; 12662 } 12663 12664 int8_t Assembler::get_prefixq(Address adr, Register src) { 12665 int8_t prfx = (int8_t)(REX_W + 12666 ((int)adr.base_needs_rex()) + 12667 ((int)adr.index_needs_rex() << 1) + 12668 ((int)(src->encoding() >= 8) << 2)); 12669 #ifdef ASSERT 12670 if (src->encoding() < 8) { 12671 if (adr.base_needs_rex()) { 12672 if (adr.index_needs_rex()) { 12673 assert(prfx == REX_WXB, "must be"); 12674 } else { 12675 assert(prfx == REX_WB, "must be"); 12676 } 12677 } else { 12678 if (adr.index_needs_rex()) { 12679 assert(prfx == REX_WX, "must be"); 12680 } else { 12681 assert(prfx == REX_W, "must be"); 12682 } 12683 } 12684 } else { 12685 if (adr.base_needs_rex()) { 12686 if (adr.index_needs_rex()) { 12687 assert(prfx == REX_WRXB, "must be"); 12688 } else { 12689 assert(prfx == REX_WRB, "must be"); 12690 } 12691 } else { 12692 if (adr.index_needs_rex()) { 12693 assert(prfx == REX_WRX, "must be"); 12694 } else { 12695 assert(prfx == REX_WR, "must be"); 12696 } 12697 } 12698 } 12699 #endif 12700 return prfx; 12701 } 12702 12703 void Assembler::prefixq(Address adr) { 12704 emit_int8(get_prefixq(adr)); 12705 } 12706 12707 void Assembler::prefixq(Address adr, Register src) { 12708 emit_int8(get_prefixq(adr, src)); 12709 } 12710 12711 void Assembler::prefixq(Address adr, XMMRegister src) { 12712 if (src->encoding() < 8) { 12713 if (adr.base_needs_rex()) { 12714 if (adr.index_needs_rex()) { 12715 prefix(REX_WXB); 12716 } else { 12717 prefix(REX_WB); 12718 } 12719 } else { 12720 if (adr.index_needs_rex()) { 12721 prefix(REX_WX); 12722 } else { 12723 prefix(REX_W); 12724 } 12725 } 12726 } else { 12727 if (adr.base_needs_rex()) { 12728 if (adr.index_needs_rex()) { 12729 prefix(REX_WRXB); 12730 } else { 12731 prefix(REX_WRB); 12732 } 12733 } else { 12734 if (adr.index_needs_rex()) { 12735 prefix(REX_WRX); 12736 } else { 12737 prefix(REX_WR); 12738 } 12739 } 12740 } 12741 } 12742 12743 int Assembler::prefixq_and_encode(int reg_enc) { 12744 if (reg_enc < 8) { 12745 prefix(REX_W); 12746 } else { 12747 prefix(REX_WB); 12748 reg_enc -= 8; 12749 } 12750 return reg_enc; 12751 } 12752 12753 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) { 12754 if (dst_enc < 8) { 12755 if (src_enc < 8) { 12756 prefix(REX_W); 12757 } else { 12758 prefix(REX_WB); 12759 src_enc -= 8; 12760 } 12761 } else { 12762 if (src_enc < 8) { 12763 prefix(REX_WR); 12764 } else { 12765 prefix(REX_WRB); 12766 src_enc -= 8; 12767 } 12768 dst_enc -= 8; 12769 } 12770 return dst_enc << 3 | src_enc; 12771 } 12772 12773 void Assembler::adcq(Register dst, int32_t imm32) { 12774 (void) prefixq_and_encode(dst->encoding()); 12775 emit_arith(0x81, 0xD0, dst, imm32); 12776 } 12777 12778 void Assembler::adcq(Register dst, Address src) { 12779 InstructionMark im(this); 12780 emit_int16(get_prefixq(src, dst), 0x13); 12781 emit_operand(dst, src, 0); 12782 } 12783 12784 void Assembler::adcq(Register dst, Register src) { 12785 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 12786 emit_arith(0x13, 0xC0, dst, src); 12787 } 12788 12789 void Assembler::addq(Address dst, int32_t imm32) { 12790 InstructionMark im(this); 12791 prefixq(dst); 12792 emit_arith_operand(0x81, rax, dst, imm32); 12793 } 12794 12795 void Assembler::addq(Address dst, Register src) { 12796 InstructionMark im(this); 12797 emit_int16(get_prefixq(dst, src), 0x01); 12798 emit_operand(src, dst, 0); 12799 } 12800 12801 void Assembler::addq(Register dst, int32_t imm32) { 12802 (void) prefixq_and_encode(dst->encoding()); 12803 emit_arith(0x81, 0xC0, dst, imm32); 12804 } 12805 12806 void Assembler::addq(Register dst, Address src) { 12807 InstructionMark im(this); 12808 emit_int16(get_prefixq(src, dst), 0x03); 12809 emit_operand(dst, src, 0); 12810 } 12811 12812 void Assembler::addq(Register dst, Register src) { 12813 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 12814 emit_arith(0x03, 0xC0, dst, src); 12815 } 12816 12817 void Assembler::adcxq(Register dst, Register src) { 12818 //assert(VM_Version::supports_adx(), "adx instructions not supported"); 12819 emit_int8(0x66); 12820 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 12821 emit_int32(0x0F, 12822 0x38, 12823 (unsigned char)0xF6, 12824 (0xC0 | encode)); 12825 } 12826 12827 void Assembler::adoxq(Register dst, Register src) { 12828 //assert(VM_Version::supports_adx(), "adx instructions not supported"); 12829 emit_int8((unsigned char)0xF3); 12830 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 12831 emit_int32(0x0F, 12832 0x38, 12833 (unsigned char)0xF6, 12834 (0xC0 | encode)); 12835 } 12836 12837 void Assembler::andq(Address dst, int32_t imm32) { 12838 InstructionMark im(this); 12839 prefixq(dst); 12840 emit_arith_operand(0x81, as_Register(4), dst, imm32); 12841 } 12842 12843 void Assembler::andq(Register dst, int32_t imm32) { 12844 (void) prefixq_and_encode(dst->encoding()); 12845 emit_arith(0x81, 0xE0, dst, imm32); 12846 } 12847 12848 void Assembler::andq(Register dst, Address src) { 12849 InstructionMark im(this); 12850 emit_int16(get_prefixq(src, dst), 0x23); 12851 emit_operand(dst, src, 0); 12852 } 12853 12854 void Assembler::andq(Register dst, Register src) { 12855 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 12856 emit_arith(0x23, 0xC0, dst, src); 12857 } 12858 12859 void Assembler::andq(Address dst, Register src) { 12860 InstructionMark im(this); 12861 emit_int16(get_prefixq(dst, src), 0x21); 12862 emit_operand(src, dst, 0); 12863 } 12864 12865 void Assembler::andnq(Register dst, Register src1, Register src2) { 12866 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12867 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12868 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12869 emit_int16((unsigned char)0xF2, (0xC0 | encode)); 12870 } 12871 12872 void Assembler::andnq(Register dst, Register src1, Address src2) { 12873 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12874 InstructionMark im(this); 12875 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12876 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12877 emit_int8((unsigned char)0xF2); 12878 emit_operand(dst, src2, 0); 12879 } 12880 12881 void Assembler::bsfq(Register dst, Register src) { 12882 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 12883 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode)); 12884 } 12885 12886 void Assembler::bsrq(Register dst, Register src) { 12887 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 12888 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode)); 12889 } 12890 12891 void Assembler::bswapq(Register reg) { 12892 int encode = prefixq_and_encode(reg->encoding()); 12893 emit_int16(0x0F, (0xC8 | encode)); 12894 } 12895 12896 void Assembler::blsiq(Register dst, Register src) { 12897 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12898 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12899 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12900 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 12901 } 12902 12903 void Assembler::blsiq(Register dst, Address src) { 12904 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12905 InstructionMark im(this); 12906 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12907 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12908 emit_int8((unsigned char)0xF3); 12909 emit_operand(rbx, src, 0); 12910 } 12911 12912 void Assembler::blsmskq(Register dst, Register src) { 12913 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12914 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12915 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12916 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 12917 } 12918 12919 void Assembler::blsmskq(Register dst, Address src) { 12920 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12921 InstructionMark im(this); 12922 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12923 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12924 emit_int8((unsigned char)0xF3); 12925 emit_operand(rdx, src, 0); 12926 } 12927 12928 void Assembler::blsrq(Register dst, Register src) { 12929 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12930 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12931 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12932 emit_int16((unsigned char)0xF3, (0xC0 | encode)); 12933 } 12934 12935 void Assembler::blsrq(Register dst, Address src) { 12936 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); 12937 InstructionMark im(this); 12938 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 12939 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes); 12940 emit_int8((unsigned char)0xF3); 12941 emit_operand(rcx, src, 0); 12942 } 12943 12944 void Assembler::cdqq() { 12945 emit_int16(REX_W, (unsigned char)0x99); 12946 } 12947 12948 void Assembler::clflush(Address adr) { 12949 assert(VM_Version::supports_clflush(), "should do"); 12950 prefix(adr); 12951 emit_int16(0x0F, (unsigned char)0xAE); 12952 emit_operand(rdi, adr, 0); 12953 } 12954 12955 void Assembler::clflushopt(Address adr) { 12956 assert(VM_Version::supports_clflushopt(), "should do!"); 12957 // adr should be base reg only with no index or offset 12958 assert(adr.index() == noreg, "index should be noreg"); 12959 assert(adr.scale() == Address::no_scale, "scale should be no_scale"); 12960 assert(adr.disp() == 0, "displacement should be 0"); 12961 // instruction prefix is 0x66 12962 emit_int8(0x66); 12963 prefix(adr); 12964 // opcode family is 0x0F 0xAE 12965 emit_int16(0x0F, (unsigned char)0xAE); 12966 // extended opcode byte is 7 == rdi 12967 emit_operand(rdi, adr, 0); 12968 } 12969 12970 void Assembler::clwb(Address adr) { 12971 assert(VM_Version::supports_clwb(), "should do!"); 12972 // adr should be base reg only with no index or offset 12973 assert(adr.index() == noreg, "index should be noreg"); 12974 assert(adr.scale() == Address::no_scale, "scale should be no_scale"); 12975 assert(adr.disp() == 0, "displacement should be 0"); 12976 // instruction prefix is 0x66 12977 emit_int8(0x66); 12978 prefix(adr); 12979 // opcode family is 0x0f 0xAE 12980 emit_int16(0x0F, (unsigned char)0xAE); 12981 // extended opcode byte is 6 == rsi 12982 emit_operand(rsi, adr, 0); 12983 } 12984 12985 void Assembler::cmovq(Condition cc, Register dst, Register src) { 12986 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 12987 emit_int24(0x0F, (0x40 | cc), (0xC0 | encode)); 12988 } 12989 12990 void Assembler::cmovq(Condition cc, Register dst, Address src) { 12991 InstructionMark im(this); 12992 emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc)); 12993 emit_operand(dst, src, 0); 12994 } 12995 12996 void Assembler::cmpq(Address dst, int32_t imm32) { 12997 InstructionMark im(this); 12998 prefixq(dst); 12999 emit_arith_operand(0x81, as_Register(7), dst, imm32); 13000 } 13001 13002 void Assembler::cmpq(Register dst, int32_t imm32) { 13003 (void) prefixq_and_encode(dst->encoding()); 13004 emit_arith(0x81, 0xF8, dst, imm32); 13005 } 13006 13007 void Assembler::cmpq(Address dst, Register src) { 13008 InstructionMark im(this); 13009 emit_int16(get_prefixq(dst, src), 0x39); 13010 emit_operand(src, dst, 0); 13011 } 13012 13013 void Assembler::cmpq(Register dst, Register src) { 13014 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13015 emit_arith(0x3B, 0xC0, dst, src); 13016 } 13017 13018 void Assembler::cmpq(Register dst, Address src) { 13019 InstructionMark im(this); 13020 emit_int16(get_prefixq(src, dst), 0x3B); 13021 emit_operand(dst, src, 0); 13022 } 13023 13024 void Assembler::cmpxchgq(Register reg, Address adr) { 13025 InstructionMark im(this); 13026 emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1); 13027 emit_operand(reg, adr, 0); 13028 } 13029 13030 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) { 13031 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13032 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13033 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13034 emit_int16(0x2A, (0xC0 | encode)); 13035 } 13036 13037 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) { 13038 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13039 InstructionMark im(this); 13040 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13041 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 13042 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13043 emit_int8(0x2A); 13044 emit_operand(dst, src, 0); 13045 } 13046 13047 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) { 13048 NOT_LP64(assert(VM_Version::supports_sse(), "")); 13049 InstructionMark im(this); 13050 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13051 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit); 13052 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 13053 emit_int8(0x2A); 13054 emit_operand(dst, src, 0); 13055 } 13056 13057 void Assembler::cvttsd2siq(Register dst, Address src) { 13058 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13059 // F2 REX.W 0F 2C /r 13060 // CVTTSD2SI r64, xmm1/m64 13061 InstructionMark im(this); 13062 emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C); 13063 emit_operand(dst, src, 0); 13064 } 13065 13066 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 13067 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13068 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13069 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13070 emit_int16(0x2C, (0xC0 | encode)); 13071 } 13072 13073 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 13074 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13075 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13076 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes); 13077 emit_int16(0x2D, (0xC0 | encode)); 13078 } 13079 13080 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 13081 NOT_LP64(assert(VM_Version::supports_sse(), "")); 13082 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13083 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes); 13084 emit_int16(0x2C, (0xC0 | encode)); 13085 } 13086 13087 void Assembler::decl(Register dst) { 13088 // Don't use it directly. Use MacroAssembler::decrementl() instead. 13089 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode) 13090 int encode = prefix_and_encode(dst->encoding()); 13091 emit_int16((unsigned char)0xFF, (0xC8 | encode)); 13092 } 13093 13094 void Assembler::decq(Register dst) { 13095 // Don't use it directly. Use MacroAssembler::decrementq() instead. 13096 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 13097 int encode = prefixq_and_encode(dst->encoding()); 13098 emit_int16((unsigned char)0xFF, 0xC8 | encode); 13099 } 13100 13101 void Assembler::decq(Address dst) { 13102 // Don't use it directly. Use MacroAssembler::decrementq() instead. 13103 InstructionMark im(this); 13104 emit_int16(get_prefixq(dst), (unsigned char)0xFF); 13105 emit_operand(rcx, dst, 0); 13106 } 13107 13108 void Assembler::fxrstor(Address src) { 13109 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE); 13110 emit_operand(as_Register(1), src, 0); 13111 } 13112 13113 void Assembler::xrstor(Address src) { 13114 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE); 13115 emit_operand(as_Register(5), src, 0); 13116 } 13117 13118 void Assembler::fxsave(Address dst) { 13119 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE); 13120 emit_operand(as_Register(0), dst, 0); 13121 } 13122 13123 void Assembler::xsave(Address dst) { 13124 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE); 13125 emit_operand(as_Register(4), dst, 0); 13126 } 13127 13128 void Assembler::idivq(Register src) { 13129 int encode = prefixq_and_encode(src->encoding()); 13130 emit_int16((unsigned char)0xF7, (0xF8 | encode)); 13131 } 13132 13133 void Assembler::divq(Register src) { 13134 int encode = prefixq_and_encode(src->encoding()); 13135 emit_int16((unsigned char)0xF7, (0xF0 | encode)); 13136 } 13137 13138 void Assembler::imulq(Register dst, Register src) { 13139 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13140 emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode)); 13141 } 13142 13143 void Assembler::imulq(Register src) { 13144 int encode = prefixq_and_encode(src->encoding()); 13145 emit_int16((unsigned char)0xF7, (0xE8 | encode)); 13146 } 13147 13148 void Assembler::imulq(Register dst, Address src, int32_t value) { 13149 InstructionMark im(this); 13150 prefixq(src, dst); 13151 if (is8bit(value)) { 13152 emit_int8((unsigned char)0x6B); 13153 emit_operand(dst, src, 1); 13154 emit_int8(value); 13155 } else { 13156 emit_int8((unsigned char)0x69); 13157 emit_operand(dst, src, 4); 13158 emit_int32(value); 13159 } 13160 } 13161 13162 void Assembler::imulq(Register dst, Register src, int value) { 13163 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13164 if (is8bit(value)) { 13165 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF)); 13166 } else { 13167 emit_int16(0x69, (0xC0 | encode)); 13168 emit_int32(value); 13169 } 13170 } 13171 13172 void Assembler::imulq(Register dst, Address src) { 13173 InstructionMark im(this); 13174 emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF); 13175 emit_operand(dst, src, 0); 13176 } 13177 13178 void Assembler::incl(Register dst) { 13179 // Don't use it directly. Use MacroAssembler::incrementl() instead. 13180 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 13181 int encode = prefix_and_encode(dst->encoding()); 13182 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 13183 } 13184 13185 void Assembler::incq(Register dst) { 13186 // Don't use it directly. Use MacroAssembler::incrementq() instead. 13187 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode) 13188 int encode = prefixq_and_encode(dst->encoding()); 13189 emit_int16((unsigned char)0xFF, (0xC0 | encode)); 13190 } 13191 13192 void Assembler::incq(Address dst) { 13193 // Don't use it directly. Use MacroAssembler::incrementq() instead. 13194 InstructionMark im(this); 13195 emit_int16(get_prefixq(dst), (unsigned char)0xFF); 13196 emit_operand(rax, dst, 0); 13197 } 13198 13199 void Assembler::lea(Register dst, Address src) { 13200 leaq(dst, src); 13201 } 13202 13203 void Assembler::leaq(Register dst, Address src) { 13204 InstructionMark im(this); 13205 emit_int16(get_prefixq(src, dst), (unsigned char)0x8D); 13206 emit_operand(dst, src, 0); 13207 } 13208 13209 void Assembler::mov64(Register dst, int64_t imm64) { 13210 InstructionMark im(this); 13211 int encode = prefixq_and_encode(dst->encoding()); 13212 emit_int8(0xB8 | encode); 13213 emit_int64(imm64); 13214 } 13215 13216 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) { 13217 InstructionMark im(this); 13218 int encode = prefixq_and_encode(dst->encoding()); 13219 emit_int8(0xB8 | encode); 13220 emit_data64(imm64, rtype, format); 13221 } 13222 13223 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) { 13224 InstructionMark im(this); 13225 int encode = prefixq_and_encode(dst->encoding()); 13226 emit_int8(0xB8 | encode); 13227 emit_data64(imm64, rspec); 13228 } 13229 13230 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) { 13231 InstructionMark im(this); 13232 int encode = prefix_and_encode(dst->encoding()); 13233 emit_int8(0xB8 | encode); 13234 emit_data((int)imm32, rspec, narrow_oop_operand); 13235 } 13236 13237 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) { 13238 InstructionMark im(this); 13239 prefix(dst); 13240 emit_int8((unsigned char)0xC7); 13241 emit_operand(rax, dst, 4); 13242 emit_data((int)imm32, rspec, narrow_oop_operand); 13243 } 13244 13245 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) { 13246 InstructionMark im(this); 13247 int encode = prefix_and_encode(src1->encoding()); 13248 emit_int16((unsigned char)0x81, (0xF8 | encode)); 13249 emit_data((int)imm32, rspec, narrow_oop_operand); 13250 } 13251 13252 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) { 13253 InstructionMark im(this); 13254 prefix(src1); 13255 emit_int8((unsigned char)0x81); 13256 emit_operand(rax, src1, 4); 13257 emit_data((int)imm32, rspec, narrow_oop_operand); 13258 } 13259 13260 void Assembler::lzcntq(Register dst, Register src) { 13261 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 13262 emit_int8((unsigned char)0xF3); 13263 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13264 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode)); 13265 } 13266 13267 void Assembler::lzcntq(Register dst, Address src) { 13268 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR"); 13269 InstructionMark im(this); 13270 emit_int8((unsigned char)0xF3); 13271 prefixq(src, dst); 13272 emit_int16(0x0F, (unsigned char)0xBD); 13273 emit_operand(dst, src, 0); 13274 } 13275 13276 void Assembler::movdq(XMMRegister dst, Register src) { 13277 // table D-1 says MMX/SSE2 13278 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13279 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13280 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13281 emit_int16(0x6E, (0xC0 | encode)); 13282 } 13283 13284 void Assembler::movdq(Register dst, XMMRegister src) { 13285 // table D-1 says MMX/SSE2 13286 NOT_LP64(assert(VM_Version::supports_sse2(), "")); 13287 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false); 13288 // swap src/dst to get correct prefix 13289 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); 13290 emit_int16(0x7E, 13291 (0xC0 | encode)); 13292 } 13293 13294 void Assembler::movq(Register dst, Register src) { 13295 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13296 emit_int16((unsigned char)0x8B, 13297 (0xC0 | encode)); 13298 } 13299 13300 void Assembler::movq(Register dst, Address src) { 13301 InstructionMark im(this); 13302 emit_int16(get_prefixq(src, dst), (unsigned char)0x8B); 13303 emit_operand(dst, src, 0); 13304 } 13305 13306 void Assembler::movq(Address dst, Register src) { 13307 InstructionMark im(this); 13308 emit_int16(get_prefixq(dst, src), (unsigned char)0x89); 13309 emit_operand(src, dst, 0); 13310 } 13311 13312 void Assembler::movq(Address dst, int32_t imm32) { 13313 InstructionMark im(this); 13314 emit_int16(get_prefixq(dst), (unsigned char)0xC7); 13315 emit_operand(as_Register(0), dst, 4); 13316 emit_int32(imm32); 13317 } 13318 13319 void Assembler::movq(Register dst, int32_t imm32) { 13320 int encode = prefixq_and_encode(dst->encoding()); 13321 emit_int16((unsigned char)0xC7, (0xC0 | encode)); 13322 emit_int32(imm32); 13323 } 13324 13325 void Assembler::movsbq(Register dst, Address src) { 13326 InstructionMark im(this); 13327 emit_int24(get_prefixq(src, dst), 13328 0x0F, 13329 (unsigned char)0xBE); 13330 emit_operand(dst, src, 0); 13331 } 13332 13333 void Assembler::movsbq(Register dst, Register src) { 13334 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13335 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode)); 13336 } 13337 13338 void Assembler::movslq(Register dst, int32_t imm32) { 13339 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx) 13340 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx) 13341 // as a result we shouldn't use until tested at runtime... 13342 ShouldNotReachHere(); 13343 InstructionMark im(this); 13344 int encode = prefixq_and_encode(dst->encoding()); 13345 emit_int8(0xC7 | encode); 13346 emit_int32(imm32); 13347 } 13348 13349 void Assembler::movslq(Address dst, int32_t imm32) { 13350 assert(is_simm32(imm32), "lost bits"); 13351 InstructionMark im(this); 13352 emit_int16(get_prefixq(dst), (unsigned char)0xC7); 13353 emit_operand(rax, dst, 4); 13354 emit_int32(imm32); 13355 } 13356 13357 void Assembler::movslq(Register dst, Address src) { 13358 InstructionMark im(this); 13359 emit_int16(get_prefixq(src, dst), 0x63); 13360 emit_operand(dst, src, 0); 13361 } 13362 13363 void Assembler::movslq(Register dst, Register src) { 13364 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13365 emit_int16(0x63, (0xC0 | encode)); 13366 } 13367 13368 void Assembler::movswq(Register dst, Address src) { 13369 InstructionMark im(this); 13370 emit_int24(get_prefixq(src, dst), 13371 0x0F, 13372 (unsigned char)0xBF); 13373 emit_operand(dst, src, 0); 13374 } 13375 13376 void Assembler::movswq(Register dst, Register src) { 13377 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13378 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode)); 13379 } 13380 13381 void Assembler::movzbq(Register dst, Address src) { 13382 InstructionMark im(this); 13383 emit_int24(get_prefixq(src, dst), 13384 0x0F, 13385 (unsigned char)0xB6); 13386 emit_operand(dst, src, 0); 13387 } 13388 13389 void Assembler::movzbq(Register dst, Register src) { 13390 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13391 emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode)); 13392 } 13393 13394 void Assembler::movzwq(Register dst, Address src) { 13395 InstructionMark im(this); 13396 emit_int24(get_prefixq(src, dst), 13397 0x0F, 13398 (unsigned char)0xB7); 13399 emit_operand(dst, src, 0); 13400 } 13401 13402 void Assembler::movzwq(Register dst, Register src) { 13403 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13404 emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode)); 13405 } 13406 13407 void Assembler::mulq(Address src) { 13408 InstructionMark im(this); 13409 emit_int16(get_prefixq(src), (unsigned char)0xF7); 13410 emit_operand(rsp, src, 0); 13411 } 13412 13413 void Assembler::mulq(Register src) { 13414 int encode = prefixq_and_encode(src->encoding()); 13415 emit_int16((unsigned char)0xF7, (0xE0 | encode)); 13416 } 13417 13418 void Assembler::mulxq(Register dst1, Register dst2, Register src) { 13419 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13420 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13421 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes); 13422 emit_int16((unsigned char)0xF6, (0xC0 | encode)); 13423 } 13424 13425 void Assembler::negq(Register dst) { 13426 int encode = prefixq_and_encode(dst->encoding()); 13427 emit_int16((unsigned char)0xF7, (0xD8 | encode)); 13428 } 13429 13430 void Assembler::negq(Address dst) { 13431 InstructionMark im(this); 13432 emit_int16(get_prefixq(dst), (unsigned char)0xF7); 13433 emit_operand(as_Register(3), dst, 0); 13434 } 13435 13436 void Assembler::notq(Register dst) { 13437 int encode = prefixq_and_encode(dst->encoding()); 13438 emit_int16((unsigned char)0xF7, (0xD0 | encode)); 13439 } 13440 13441 void Assembler::btsq(Address dst, int imm8) { 13442 assert(isByte(imm8), "not a byte"); 13443 InstructionMark im(this); 13444 emit_int24(get_prefixq(dst), 13445 0x0F, 13446 (unsigned char)0xBA); 13447 emit_operand(rbp /* 5 */, dst, 1); 13448 emit_int8(imm8); 13449 } 13450 13451 void Assembler::btrq(Address dst, int imm8) { 13452 assert(isByte(imm8), "not a byte"); 13453 InstructionMark im(this); 13454 emit_int24(get_prefixq(dst), 13455 0x0F, 13456 (unsigned char)0xBA); 13457 emit_operand(rsi /* 6 */, dst, 1); 13458 emit_int8(imm8); 13459 } 13460 13461 void Assembler::orq(Address dst, int32_t imm32) { 13462 InstructionMark im(this); 13463 prefixq(dst); 13464 emit_arith_operand(0x81, as_Register(1), dst, imm32); 13465 } 13466 13467 void Assembler::orq(Address dst, Register src) { 13468 InstructionMark im(this); 13469 emit_int16(get_prefixq(dst, src), (unsigned char)0x09); 13470 emit_operand(src, dst, 0); 13471 } 13472 13473 void Assembler::orq(Register dst, int32_t imm32) { 13474 (void) prefixq_and_encode(dst->encoding()); 13475 emit_arith(0x81, 0xC8, dst, imm32); 13476 } 13477 13478 void Assembler::orq_imm32(Register dst, int32_t imm32) { 13479 (void) prefixq_and_encode(dst->encoding()); 13480 emit_arith_imm32(0x81, 0xC8, dst, imm32); 13481 } 13482 13483 void Assembler::orq(Register dst, Address src) { 13484 InstructionMark im(this); 13485 emit_int16(get_prefixq(src, dst), 0x0B); 13486 emit_operand(dst, src, 0); 13487 } 13488 13489 void Assembler::orq(Register dst, Register src) { 13490 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13491 emit_arith(0x0B, 0xC0, dst, src); 13492 } 13493 13494 void Assembler::popcntq(Register dst, Address src) { 13495 assert(VM_Version::supports_popcnt(), "must support"); 13496 InstructionMark im(this); 13497 emit_int32((unsigned char)0xF3, 13498 get_prefixq(src, dst), 13499 0x0F, 13500 (unsigned char)0xB8); 13501 emit_operand(dst, src, 0); 13502 } 13503 13504 void Assembler::popcntq(Register dst, Register src) { 13505 assert(VM_Version::supports_popcnt(), "must support"); 13506 emit_int8((unsigned char)0xF3); 13507 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13508 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode)); 13509 } 13510 13511 void Assembler::popq(Address dst) { 13512 InstructionMark im(this); 13513 emit_int16(get_prefixq(dst), (unsigned char)0x8F); 13514 emit_operand(rax, dst, 0); 13515 } 13516 13517 void Assembler::popq(Register dst) { 13518 emit_int8((unsigned char)0x58 | dst->encoding()); 13519 } 13520 13521 // Precomputable: popa, pusha, vzeroupper 13522 13523 // The result of these routines are invariant from one invocation to another 13524 // invocation for the duration of a run. Caching the result on bootstrap 13525 // and copying it out on subsequent invocations can thus be beneficial 13526 static bool precomputed = false; 13527 13528 static u_char* popa_code = NULL; 13529 static int popa_len = 0; 13530 13531 static u_char* pusha_code = NULL; 13532 static int pusha_len = 0; 13533 13534 static u_char* vzup_code = NULL; 13535 static int vzup_len = 0; 13536 13537 void Assembler::precompute_instructions() { 13538 assert(!Universe::is_fully_initialized(), "must still be single threaded"); 13539 guarantee(!precomputed, "only once"); 13540 precomputed = true; 13541 ResourceMark rm; 13542 13543 // Make a temporary buffer big enough for the routines we're capturing 13544 int size = 256; 13545 char* tmp_code = NEW_RESOURCE_ARRAY(char, size); 13546 CodeBuffer buffer((address)tmp_code, size); 13547 MacroAssembler masm(&buffer); 13548 13549 address begin_popa = masm.code_section()->end(); 13550 masm.popa_uncached(); 13551 address end_popa = masm.code_section()->end(); 13552 masm.pusha_uncached(); 13553 address end_pusha = masm.code_section()->end(); 13554 masm.vzeroupper_uncached(); 13555 address end_vzup = masm.code_section()->end(); 13556 13557 // Save the instructions to permanent buffers. 13558 popa_len = (int)(end_popa - begin_popa); 13559 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal); 13560 memcpy(popa_code, begin_popa, popa_len); 13561 13562 pusha_len = (int)(end_pusha - end_popa); 13563 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal); 13564 memcpy(pusha_code, end_popa, pusha_len); 13565 13566 vzup_len = (int)(end_vzup - end_pusha); 13567 if (vzup_len > 0) { 13568 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal); 13569 memcpy(vzup_code, end_pusha, vzup_len); 13570 } else { 13571 vzup_code = pusha_code; // dummy 13572 } 13573 13574 assert(masm.code()->total_oop_size() == 0 && 13575 masm.code()->total_metadata_size() == 0 && 13576 masm.code()->total_relocation_size() == 0, 13577 "pre-computed code can't reference oops, metadata or contain relocations"); 13578 } 13579 13580 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) { 13581 assert(src != NULL, "code to copy must have been pre-computed"); 13582 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough"); 13583 address end = code_section->end(); 13584 memcpy(end, src, src_len); 13585 code_section->set_end(end + src_len); 13586 } 13587 13588 void Assembler::popa() { // 64bit 13589 emit_copy(code_section(), popa_code, popa_len); 13590 } 13591 13592 void Assembler::popa_uncached() { // 64bit 13593 movq(r15, Address(rsp, 0)); 13594 movq(r14, Address(rsp, wordSize)); 13595 movq(r13, Address(rsp, 2 * wordSize)); 13596 movq(r12, Address(rsp, 3 * wordSize)); 13597 movq(r11, Address(rsp, 4 * wordSize)); 13598 movq(r10, Address(rsp, 5 * wordSize)); 13599 movq(r9, Address(rsp, 6 * wordSize)); 13600 movq(r8, Address(rsp, 7 * wordSize)); 13601 movq(rdi, Address(rsp, 8 * wordSize)); 13602 movq(rsi, Address(rsp, 9 * wordSize)); 13603 movq(rbp, Address(rsp, 10 * wordSize)); 13604 // Skip rsp as it is restored automatically to the value 13605 // before the corresponding pusha when popa is done. 13606 movq(rbx, Address(rsp, 12 * wordSize)); 13607 movq(rdx, Address(rsp, 13 * wordSize)); 13608 movq(rcx, Address(rsp, 14 * wordSize)); 13609 movq(rax, Address(rsp, 15 * wordSize)); 13610 13611 addq(rsp, 16 * wordSize); 13612 } 13613 13614 // Does not actually store the value of rsp on the stack. 13615 // The slot for rsp just contains an arbitrary value. 13616 void Assembler::pusha() { // 64bit 13617 emit_copy(code_section(), pusha_code, pusha_len); 13618 } 13619 13620 // Does not actually store the value of rsp on the stack. 13621 // The slot for rsp just contains an arbitrary value. 13622 void Assembler::pusha_uncached() { // 64bit 13623 subq(rsp, 16 * wordSize); 13624 13625 movq(Address(rsp, 15 * wordSize), rax); 13626 movq(Address(rsp, 14 * wordSize), rcx); 13627 movq(Address(rsp, 13 * wordSize), rdx); 13628 movq(Address(rsp, 12 * wordSize), rbx); 13629 // Skip rsp as the value is normally not used. There are a few places where 13630 // the original value of rsp needs to be known but that can be computed 13631 // from the value of rsp immediately after pusha (rsp + 16 * wordSize). 13632 movq(Address(rsp, 10 * wordSize), rbp); 13633 movq(Address(rsp, 9 * wordSize), rsi); 13634 movq(Address(rsp, 8 * wordSize), rdi); 13635 movq(Address(rsp, 7 * wordSize), r8); 13636 movq(Address(rsp, 6 * wordSize), r9); 13637 movq(Address(rsp, 5 * wordSize), r10); 13638 movq(Address(rsp, 4 * wordSize), r11); 13639 movq(Address(rsp, 3 * wordSize), r12); 13640 movq(Address(rsp, 2 * wordSize), r13); 13641 movq(Address(rsp, wordSize), r14); 13642 movq(Address(rsp, 0), r15); 13643 } 13644 13645 void Assembler::vzeroupper() { 13646 emit_copy(code_section(), vzup_code, vzup_len); 13647 } 13648 13649 void Assembler::vzeroall() { 13650 assert(VM_Version::supports_avx(), "requires AVX"); 13651 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13652 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes); 13653 emit_int8(0x77); 13654 } 13655 13656 void Assembler::pushq(Address src) { 13657 InstructionMark im(this); 13658 emit_int16(get_prefixq(src), (unsigned char)0xFF); 13659 emit_operand(rsi, src, 0); 13660 } 13661 13662 void Assembler::rclq(Register dst, int imm8) { 13663 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13664 int encode = prefixq_and_encode(dst->encoding()); 13665 if (imm8 == 1) { 13666 emit_int16((unsigned char)0xD1, (0xD0 | encode)); 13667 } else { 13668 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8); 13669 } 13670 } 13671 13672 void Assembler::rcrq(Register dst, int imm8) { 13673 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13674 int encode = prefixq_and_encode(dst->encoding()); 13675 if (imm8 == 1) { 13676 emit_int16((unsigned char)0xD1, (0xD8 | encode)); 13677 } else { 13678 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8); 13679 } 13680 } 13681 13682 void Assembler::rorxl(Register dst, Register src, int imm8) { 13683 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13684 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13685 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 13686 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8); 13687 } 13688 13689 void Assembler::rorxl(Register dst, Address src, int imm8) { 13690 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13691 InstructionMark im(this); 13692 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13693 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 13694 emit_int8((unsigned char)0xF0); 13695 emit_operand(dst, src, 1); 13696 emit_int8(imm8); 13697 } 13698 13699 void Assembler::rorxq(Register dst, Register src, int imm8) { 13700 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13701 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13702 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 13703 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8); 13704 } 13705 13706 void Assembler::rorxq(Register dst, Address src, int imm8) { 13707 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported"); 13708 InstructionMark im(this); 13709 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); 13710 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes); 13711 emit_int8((unsigned char)0xF0); 13712 emit_operand(dst, src, 1); 13713 emit_int8(imm8); 13714 } 13715 13716 #ifdef _LP64 13717 void Assembler::salq(Address dst, int imm8) { 13718 InstructionMark im(this); 13719 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13720 if (imm8 == 1) { 13721 emit_int16(get_prefixq(dst), (unsigned char)0xD1); 13722 emit_operand(as_Register(4), dst, 0); 13723 } 13724 else { 13725 emit_int16(get_prefixq(dst), (unsigned char)0xC1); 13726 emit_operand(as_Register(4), dst, 1); 13727 emit_int8(imm8); 13728 } 13729 } 13730 13731 void Assembler::salq(Address dst) { 13732 InstructionMark im(this); 13733 emit_int16(get_prefixq(dst), (unsigned char)0xD3); 13734 emit_operand(as_Register(4), dst, 0); 13735 } 13736 13737 void Assembler::salq(Register dst, int imm8) { 13738 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13739 int encode = prefixq_and_encode(dst->encoding()); 13740 if (imm8 == 1) { 13741 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 13742 } else { 13743 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 13744 } 13745 } 13746 13747 void Assembler::salq(Register dst) { 13748 int encode = prefixq_and_encode(dst->encoding()); 13749 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 13750 } 13751 13752 void Assembler::sarq(Address dst, int imm8) { 13753 InstructionMark im(this); 13754 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13755 if (imm8 == 1) { 13756 emit_int16(get_prefixq(dst), (unsigned char)0xD1); 13757 emit_operand(as_Register(7), dst, 0); 13758 } 13759 else { 13760 emit_int16(get_prefixq(dst), (unsigned char)0xC1); 13761 emit_operand(as_Register(7), dst, 1); 13762 emit_int8(imm8); 13763 } 13764 } 13765 13766 void Assembler::sarq(Address dst) { 13767 InstructionMark im(this); 13768 emit_int16(get_prefixq(dst), (unsigned char)0xD3); 13769 emit_operand(as_Register(7), dst, 0); 13770 } 13771 13772 void Assembler::sarq(Register dst, int imm8) { 13773 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13774 int encode = prefixq_and_encode(dst->encoding()); 13775 if (imm8 == 1) { 13776 emit_int16((unsigned char)0xD1, (0xF8 | encode)); 13777 } else { 13778 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8); 13779 } 13780 } 13781 13782 void Assembler::sarq(Register dst) { 13783 int encode = prefixq_and_encode(dst->encoding()); 13784 emit_int16((unsigned char)0xD3, (0xF8 | encode)); 13785 } 13786 #endif 13787 13788 void Assembler::sbbq(Address dst, int32_t imm32) { 13789 InstructionMark im(this); 13790 prefixq(dst); 13791 emit_arith_operand(0x81, rbx, dst, imm32); 13792 } 13793 13794 void Assembler::sbbq(Register dst, int32_t imm32) { 13795 (void) prefixq_and_encode(dst->encoding()); 13796 emit_arith(0x81, 0xD8, dst, imm32); 13797 } 13798 13799 void Assembler::sbbq(Register dst, Address src) { 13800 InstructionMark im(this); 13801 emit_int16(get_prefixq(src, dst), 0x1B); 13802 emit_operand(dst, src, 0); 13803 } 13804 13805 void Assembler::sbbq(Register dst, Register src) { 13806 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13807 emit_arith(0x1B, 0xC0, dst, src); 13808 } 13809 13810 void Assembler::shlq(Register dst, int imm8) { 13811 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13812 int encode = prefixq_and_encode(dst->encoding()); 13813 if (imm8 == 1) { 13814 emit_int16((unsigned char)0xD1, (0xE0 | encode)); 13815 } else { 13816 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8); 13817 } 13818 } 13819 13820 void Assembler::shlq(Register dst) { 13821 int encode = prefixq_and_encode(dst->encoding()); 13822 emit_int16((unsigned char)0xD3, (0xE0 | encode)); 13823 } 13824 13825 void Assembler::shrq(Register dst, int imm8) { 13826 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13827 int encode = prefixq_and_encode(dst->encoding()); 13828 if (imm8 == 1) { 13829 emit_int16((unsigned char)0xD1, (0xE8 | encode)); 13830 } 13831 else { 13832 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8); 13833 } 13834 } 13835 13836 void Assembler::shrq(Register dst) { 13837 int encode = prefixq_and_encode(dst->encoding()); 13838 emit_int16((unsigned char)0xD3, 0xE8 | encode); 13839 } 13840 13841 void Assembler::shrq(Address dst) { 13842 InstructionMark im(this); 13843 emit_int16(get_prefixq(dst), (unsigned char)0xD3); 13844 emit_operand(as_Register(5), dst, 0); 13845 } 13846 13847 void Assembler::shrq(Address dst, int imm8) { 13848 InstructionMark im(this); 13849 assert(isShiftCount(imm8 >> 1), "illegal shift count"); 13850 if (imm8 == 1) { 13851 emit_int16(get_prefixq(dst), (unsigned char)0xD1); 13852 emit_operand(as_Register(5), dst, 0); 13853 } 13854 else { 13855 emit_int16(get_prefixq(dst), (unsigned char)0xC1); 13856 emit_operand(as_Register(5), dst, 1); 13857 emit_int8(imm8); 13858 } 13859 } 13860 13861 void Assembler::subq(Address dst, int32_t imm32) { 13862 InstructionMark im(this); 13863 prefixq(dst); 13864 emit_arith_operand(0x81, rbp, dst, imm32); 13865 } 13866 13867 void Assembler::subq(Address dst, Register src) { 13868 InstructionMark im(this); 13869 emit_int16(get_prefixq(dst, src), 0x29); 13870 emit_operand(src, dst, 0); 13871 } 13872 13873 void Assembler::subq(Register dst, int32_t imm32) { 13874 (void) prefixq_and_encode(dst->encoding()); 13875 emit_arith(0x81, 0xE8, dst, imm32); 13876 } 13877 13878 // Force generation of a 4 byte immediate value even if it fits into 8bit 13879 void Assembler::subq_imm32(Register dst, int32_t imm32) { 13880 (void) prefixq_and_encode(dst->encoding()); 13881 emit_arith_imm32(0x81, 0xE8, dst, imm32); 13882 } 13883 13884 void Assembler::subq(Register dst, Address src) { 13885 InstructionMark im(this); 13886 emit_int16(get_prefixq(src, dst), 0x2B); 13887 emit_operand(dst, src, 0); 13888 } 13889 13890 void Assembler::subq(Register dst, Register src) { 13891 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13892 emit_arith(0x2B, 0xC0, dst, src); 13893 } 13894 13895 void Assembler::testq(Address dst, int32_t imm32) { 13896 InstructionMark im(this); 13897 emit_int16(get_prefixq(dst), (unsigned char)0xF7); 13898 emit_operand(as_Register(0), dst, 4); 13899 emit_int32(imm32); 13900 } 13901 13902 void Assembler::testq(Register dst, int32_t imm32) { 13903 // not using emit_arith because test 13904 // doesn't support sign-extension of 13905 // 8bit operands 13906 if (dst == rax) { 13907 prefix(REX_W); 13908 emit_int8((unsigned char)0xA9); 13909 emit_int32(imm32); 13910 } else { 13911 int encode = dst->encoding(); 13912 encode = prefixq_and_encode(encode); 13913 emit_int16((unsigned char)0xF7, (0xC0 | encode)); 13914 emit_int32(imm32); 13915 } 13916 } 13917 13918 void Assembler::testq(Register dst, Register src) { 13919 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13920 emit_arith(0x85, 0xC0, dst, src); 13921 } 13922 13923 void Assembler::testq(Register dst, Address src) { 13924 InstructionMark im(this); 13925 emit_int16(get_prefixq(src, dst), (unsigned char)0x85); 13926 emit_operand(dst, src, 0); 13927 } 13928 13929 void Assembler::xaddq(Address dst, Register src) { 13930 InstructionMark im(this); 13931 emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1); 13932 emit_operand(src, dst, 0); 13933 } 13934 13935 void Assembler::xchgq(Register dst, Address src) { 13936 InstructionMark im(this); 13937 emit_int16(get_prefixq(src, dst), (unsigned char)0x87); 13938 emit_operand(dst, src, 0); 13939 } 13940 13941 void Assembler::xchgq(Register dst, Register src) { 13942 int encode = prefixq_and_encode(dst->encoding(), src->encoding()); 13943 emit_int16((unsigned char)0x87, (0xc0 | encode)); 13944 } 13945 13946 void Assembler::xorq(Register dst, Register src) { 13947 (void) prefixq_and_encode(dst->encoding(), src->encoding()); 13948 emit_arith(0x33, 0xC0, dst, src); 13949 } 13950 13951 void Assembler::xorq(Register dst, Address src) { 13952 InstructionMark im(this); 13953 emit_int16(get_prefixq(src, dst), 0x33); 13954 emit_operand(dst, src, 0); 13955 } 13956 13957 void Assembler::xorq(Register dst, int32_t imm32) { 13958 (void) prefixq_and_encode(dst->encoding()); 13959 emit_arith(0x81, 0xF0, dst, imm32); 13960 } 13961 13962 void Assembler::xorq(Address dst, int32_t imm32) { 13963 InstructionMark im(this); 13964 prefixq(dst); 13965 emit_arith_operand(0x81, as_Register(6), dst, imm32); 13966 } 13967 13968 void Assembler::xorq(Address dst, Register src) { 13969 InstructionMark im(this); 13970 emit_int16(get_prefixq(dst, src), 0x31); 13971 emit_operand(src, dst, 0); 13972 } 13973 13974 #endif // !LP64 13975 13976 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) { 13977 if (VM_Version::supports_evex()) { 13978 _tuple_type = tuple_type; 13979 _input_size_in_bits = input_size_in_bits; 13980 } 13981 }