1 /*
2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/assembler.inline.hpp"
27 #include "asm/codeBuffer.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/shared/cardTableBarrierSet.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "memory/universe.hpp"
33 #include "prims/methodHandles.hpp"
34 #include "runtime/objectMonitor.hpp"
35 #include "runtime/os.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "runtime/vm_version.hpp"
39 #include "utilities/checkedCast.hpp"
40 #include "utilities/macros.hpp"
41
42 #ifdef PRODUCT
43 #define BLOCK_COMMENT(str) /* nothing */
44 #define STOP(error) stop(error)
45 #else
46 #define BLOCK_COMMENT(str) block_comment(str)
47 #define STOP(error) block_comment(error); stop(error)
48 #endif
49
50 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
51 // Implementation of AddressLiteral
52
53 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
54 static const unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
55 // -----------------Table 4.5 -------------------- //
56 16, 32, 64, // EVEX_FV(0)
57 4, 4, 4, // EVEX_FV(1) - with Evex.b
58 16, 32, 64, // EVEX_FV(2) - with Evex.w
59 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
60 8, 16, 32, // EVEX_HV(0)
61 4, 4, 4, // EVEX_HV(1) - with Evex.b
62 // -----------------Table 4.6 -------------------- //
63 16, 32, 64, // EVEX_FVM(0)
64 1, 1, 1, // EVEX_T1S(0)
65 2, 2, 2, // EVEX_T1S(1)
66 4, 4, 4, // EVEX_T1S(2)
67 8, 8, 8, // EVEX_T1S(3)
68 4, 4, 4, // EVEX_T1F(0)
69 8, 8, 8, // EVEX_T1F(1)
70 8, 8, 8, // EVEX_T2(0)
71 0, 16, 16, // EVEX_T2(1)
72 0, 16, 16, // EVEX_T4(0)
73 0, 0, 32, // EVEX_T4(1)
74 0, 0, 32, // EVEX_T8(0)
75 8, 16, 32, // EVEX_HVM(0)
76 4, 8, 16, // EVEX_QVM(0)
77 2, 4, 8, // EVEX_OVM(0)
78 16, 16, 16, // EVEX_M128(0)
79 8, 32, 64, // EVEX_DUP(0)
80 1, 1, 1, // EVEX_NOSCALE(0)
81 0, 0, 0 // EVEX_ETUP
82 };
83
84 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
85 _is_lval = false;
86 _target = target;
87 switch (rtype) {
88 case relocInfo::oop_type:
89 case relocInfo::metadata_type:
90 // Oops are a special case. Normally they would be their own section
91 // but in cases like icBuffer they are literals in the code stream that
92 // we don't have a section for. We use none so that we get a literal address
93 // which is always patchable.
94 break;
95 case relocInfo::external_word_type:
96 _rspec = external_word_Relocation::spec(target);
97 break;
98 case relocInfo::internal_word_type:
99 _rspec = internal_word_Relocation::spec(target);
100 break;
101 case relocInfo::opt_virtual_call_type:
102 _rspec = opt_virtual_call_Relocation::spec();
103 break;
104 case relocInfo::static_call_type:
105 _rspec = static_call_Relocation::spec();
106 break;
107 case relocInfo::runtime_call_type:
108 _rspec = runtime_call_Relocation::spec();
109 break;
110 case relocInfo::poll_type:
111 case relocInfo::poll_return_type:
112 _rspec = Relocation::spec_simple(rtype);
113 break;
114 case relocInfo::none:
115 break;
116 default:
117 ShouldNotReachHere();
118 break;
119 }
120 }
121
122 // Implementation of Address
123
124 Address Address::make_array(ArrayAddress adr) {
125 // Not implementable on 64bit machines
126 // Should have been handled higher up the call chain.
127 ShouldNotReachHere();
128 return Address();
129 }
130
131 // exceedingly dangerous constructor
132 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
133 _base = noreg;
134 _index = noreg;
135 _scale = no_scale;
136 _disp = disp;
137 _xmmindex = xnoreg;
138 _isxmmindex = false;
139 switch (rtype) {
140 case relocInfo::external_word_type:
141 _rspec = external_word_Relocation::spec(loc);
142 break;
143 case relocInfo::internal_word_type:
144 _rspec = internal_word_Relocation::spec(loc);
145 break;
146 case relocInfo::runtime_call_type:
147 // HMM
148 _rspec = runtime_call_Relocation::spec();
149 break;
150 case relocInfo::poll_type:
151 case relocInfo::poll_return_type:
152 _rspec = Relocation::spec_simple(rtype);
153 break;
154 case relocInfo::none:
155 break;
156 default:
157 ShouldNotReachHere();
158 }
159 }
160
161
162 // Convert the raw encoding form into the form expected by the constructor for
163 // Address. An index of 4 (rsp) corresponds to having no index, so convert
164 // that to noreg for the Address constructor.
165 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
166 RelocationHolder rspec = RelocationHolder::none;
167 if (disp_reloc != relocInfo::none) {
168 rspec = Relocation::spec_simple(disp_reloc);
169 }
170 bool valid_index = index != rsp->encoding();
171 if (valid_index) {
172 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
173 madr._rspec = rspec;
174 return madr;
175 } else {
176 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
177 madr._rspec = rspec;
178 return madr;
179 }
180 }
181
182 // Implementation of Assembler
183
184 int AbstractAssembler::code_fill_byte() {
185 return (u_char)'\xF4'; // hlt
186 }
187
188 void Assembler::init_attributes(void) {
189 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
190 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
191 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
192 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
193 _attributes = nullptr;
194 }
195
196 void Assembler::set_attributes(InstructionAttr* attributes) {
197 // Record the assembler in the attributes, so the attributes destructor can
198 // clear the assembler's attributes, cleaning up the otherwise dangling
199 // pointer. gcc13 has a false positive warning, because it doesn't tie that
200 // cleanup to the assignment of _attributes here.
201 attributes->set_current_assembler(this);
202 PRAGMA_DIAG_PUSH
203 PRAGMA_DANGLING_POINTER_IGNORED
204 _attributes = attributes;
205 PRAGMA_DIAG_POP
206 }
207
208 void Assembler::membar(Membar_mask_bits order_constraint) {
209 // We only have to handle StoreLoad
210 if (order_constraint & StoreLoad) {
211 // All usable chips support "locked" instructions which suffice
212 // as barriers, and are much faster than the alternative of
213 // using cpuid instruction. We use here a locked add [esp-C],0.
214 // This is conveniently otherwise a no-op except for blowing
215 // flags, and introducing a false dependency on target memory
216 // location. We can't do anything with flags, but we can avoid
217 // memory dependencies in the current method by locked-adding
218 // somewhere else on the stack. Doing [esp+C] will collide with
219 // something on stack in current method, hence we go for [esp-C].
220 // It is convenient since it is almost always in data cache, for
221 // any small C. We need to step back from SP to avoid data
222 // dependencies with other things on below SP (callee-saves, for
223 // example). Without a clear way to figure out the minimal safe
224 // distance from SP, it makes sense to step back the complete
225 // cache line, as this will also avoid possible second-order effects
226 // with locked ops against the cache line. Our choice of offset
227 // is bounded by x86 operand encoding, which should stay within
228 // [-128; +127] to have the 8-byte displacement encoding.
229 //
230 // Any change to this code may need to revisit other places in
231 // the code where this idiom is used, in particular the
232 // orderAccess code.
233
234 int offset = -VM_Version::L1_line_size();
235 if (offset < -128) {
236 offset = -128;
237 }
238
239 lock();
240 addl(Address(rsp, offset), 0);// Assert the lock# signal here
241 }
242 }
243
244 // make this go away someday
245 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
246 if (rtype == relocInfo::none)
247 emit_int32(data);
248 else
249 emit_data(data, Relocation::spec_simple(rtype), format);
250 }
251
252 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
253 assert(imm_operand == 0, "default format must be immediate in this file");
254 assert(inst_mark() != nullptr, "must be inside InstructionMark");
255 if (rspec.type() != relocInfo::none) {
256 #ifdef ASSERT
257 check_relocation(rspec, format);
258 #endif
259 // Do not use AbstractAssembler::relocate, which is not intended for
260 // embedded words. Instead, relocate to the enclosing instruction.
261
262 // hack. call32 is too wide for mask so use disp32
263 if (format == call32_operand)
264 code_section()->relocate(inst_mark(), rspec, disp32_operand);
265 else
266 code_section()->relocate(inst_mark(), rspec, format);
267 }
268 emit_int32(data);
269 }
270
271 static int encode(Register r) {
272 return r->encoding() & 7;
273 }
274
275 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
276 assert(dst->has_byte_register(), "must have byte register");
277 assert(isByte(op1) && isByte(op2), "wrong opcode");
278 assert(isByte(imm8), "not a byte");
279 assert((op1 & 0x01) == 0, "should be 8bit operation");
280 emit_int24(op1, (op2 | encode(dst)), imm8);
281 }
282
283 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32, bool optimize_rax_dst) {
284 assert(isByte(op1) && isByte(op2), "wrong opcode");
285 assert(op1 == 0x81, "Unexpected opcode");
286 if (is8bit(imm32)) {
287 emit_int24(op1 | 0x02, // set sign bit
288 op2 | encode(dst),
289 imm32 & 0xFF);
290 } else if (optimize_rax_dst && dst == rax) {
291 switch (op2) {
292 case 0xD0: emit_int8(0x15); break; // adc
293 case 0xC0: emit_int8(0x05); break; // add
294 case 0xE0: emit_int8(0x25); break; // and
295 case 0xF8: emit_int8(0x3D); break; // cmp
296 case 0xC8: emit_int8(0x0D); break; // or
297 case 0xD8: emit_int8(0x1D); break; // sbb
298 case 0xE8: emit_int8(0x2D); break; // sub
299 case 0xF0: emit_int8(0x35); break; // xor
300 default: ShouldNotReachHere();
301 }
302 emit_int32(imm32);
303 } else {
304 emit_int16(op1, (op2 | encode(dst)));
305 emit_int32(imm32);
306 }
307 }
308
309 // Force generation of a 4 byte immediate value even if it fits into 8bit
310 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
311 assert(isByte(op1) && isByte(op2), "wrong opcode");
312 assert((op1 & 0x01) == 1, "should be 32bit operation");
313 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
314 emit_int16(op1, (op2 | encode(dst)));
315 emit_int32(imm32);
316 }
317
318 // immediate-to-memory forms
319 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
320 assert((op1 & 0x01) == 1, "should be 32bit operation");
321 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
322 if (is8bit(imm32)) {
323 emit_int8(op1 | 0x02); // set sign bit
324 emit_operand(rm, adr, 1);
325 emit_int8(imm32 & 0xFF);
326 } else {
327 emit_int8(op1);
328 emit_operand(rm, adr, 4);
329 emit_int32(imm32);
330 }
331 }
332
333 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
334 assert(op1 == 0x81, "unexpected opcode");
335 emit_int8(op1);
336 emit_operand(rm, adr, 4);
337 emit_int32(imm32);
338 }
339
340 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
341 assert(isByte(op1) && isByte(op2), "wrong opcode");
342 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
343 }
344
345
346 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
347 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
348 int mod_idx = 0;
349 // We will test if the displacement fits the compressed format and if so
350 // apply the compression to the displacement iff the result is8bit.
351 if (VM_Version::supports_evex() && is_evex_inst) {
352 switch (cur_tuple_type) {
353 case EVEX_FV:
354 if ((cur_encoding & VEX_W) == VEX_W) {
355 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
356 } else {
357 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
358 }
359 break;
360
361 case EVEX_HV:
362 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
363 break;
364
365 case EVEX_FVM:
366 break;
367
368 case EVEX_T1S:
369 switch (in_size_in_bits) {
370 case EVEX_8bit:
371 break;
372
373 case EVEX_16bit:
374 mod_idx = 1;
375 break;
376
377 case EVEX_32bit:
378 mod_idx = 2;
379 break;
380
381 case EVEX_64bit:
382 mod_idx = 3;
383 break;
384 }
385 break;
386
387 case EVEX_T1F:
388 case EVEX_T2:
389 case EVEX_T4:
390 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
391 break;
392
393 case EVEX_T8:
394 break;
395
396 case EVEX_HVM:
397 break;
398
399 case EVEX_QVM:
400 break;
401
402 case EVEX_OVM:
403 break;
404
405 case EVEX_M128:
406 break;
407
408 case EVEX_DUP:
409 break;
410
411 case EVEX_NOSCALE:
412 break;
413
414 default:
415 assert(0, "no valid evex tuple_table entry");
416 break;
417 }
418
419 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
420 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
421 if ((disp % disp_factor) == 0) {
422 int new_disp = disp / disp_factor;
423 if ((-0x80 <= new_disp && new_disp < 0x80)) {
424 disp = new_disp;
425 }
426 } else {
427 return false;
428 }
429 }
430 }
431 return (-0x80 <= disp && disp < 0x80);
432 }
433
434
435 bool Assembler::emit_compressed_disp_byte(int &disp) {
436 int mod_idx = 0;
437 // We will test if the displacement fits the compressed format and if so
438 // apply the compression to the displacement iff the result is8bit.
439 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
440 int evex_encoding = _attributes->get_evex_encoding();
441 int tuple_type = _attributes->get_tuple_type();
442 switch (tuple_type) {
443 case EVEX_FV:
444 if ((evex_encoding & VEX_W) == VEX_W) {
445 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
446 } else {
447 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
448 }
449 break;
450
451 case EVEX_HV:
452 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
453 break;
454
455 case EVEX_FVM:
456 break;
457
458 case EVEX_T1S:
459 switch (_attributes->get_input_size()) {
460 case EVEX_8bit:
461 break;
462
463 case EVEX_16bit:
464 mod_idx = 1;
465 break;
466
467 case EVEX_32bit:
468 mod_idx = 2;
469 break;
470
471 case EVEX_64bit:
472 mod_idx = 3;
473 break;
474 }
475 break;
476
477 case EVEX_T1F:
478 case EVEX_T2:
479 case EVEX_T4:
480 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
481 break;
482
483 case EVEX_T8:
484 break;
485
486 case EVEX_HVM:
487 break;
488
489 case EVEX_QVM:
490 break;
491
492 case EVEX_OVM:
493 break;
494
495 case EVEX_M128:
496 break;
497
498 case EVEX_DUP:
499 break;
500
501 case EVEX_NOSCALE:
502 break;
503
504 default:
505 assert(0, "no valid evex tuple_table entry");
506 break;
507 }
508
509 int vector_len = _attributes->get_vector_len();
510 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
511 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
512 if ((disp % disp_factor) == 0) {
513 int new_disp = disp / disp_factor;
514 if (is8bit(new_disp)) {
515 disp = new_disp;
516 }
517 } else {
518 return false;
519 }
520 }
521 }
522 return is8bit(disp);
523 }
524
525 bool Assembler::needs_rex2(Register reg1, Register reg2, Register reg3) {
526 bool rex2 = (reg1->is_valid() && reg1->encoding() >= 16) ||
527 (reg2->is_valid() && reg2->encoding() >= 16) ||
528 (reg3->is_valid() && reg3->encoding() >= 16);
529 assert(!rex2 || UseAPX, "extended gpr use requires UseAPX");
530 return rex2;
531 }
532
533 #ifndef PRODUCT
534 bool Assembler::needs_evex(XMMRegister reg1, XMMRegister reg2, XMMRegister reg3) {
535 return (reg1->is_valid() && reg1->encoding() >= 16) ||
536 (reg2->is_valid() && reg2->encoding() >= 16) ||
537 (reg3->is_valid() && reg3->encoding() >= 16);
538 }
539 #endif
540
541 bool Assembler::needs_eevex(Register reg1, Register reg2, Register reg3) {
542 return needs_rex2(reg1, reg2, reg3);
543 }
544
545 bool Assembler::needs_eevex(int enc1, int enc2, int enc3) {
546 bool eevex = enc1 >= 16 || enc2 >= 16 || enc3 >=16;
547 assert(!eevex || UseAPX, "extended gpr use requires UseAPX");
548 return eevex;
549 }
550
551 static bool is_valid_encoding(int reg_enc) {
552 return reg_enc >= 0;
553 }
554
555 static int raw_encode(Register reg) {
556 assert(reg == noreg || reg->is_valid(), "sanity");
557 int reg_enc = reg->raw_encoding();
558 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
559 return reg_enc;
560 }
561
562 static int raw_encode(XMMRegister xmmreg) {
563 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
564 int xmmreg_enc = xmmreg->raw_encoding();
565 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
566 return xmmreg_enc;
567 }
568
569 static int raw_encode(KRegister kreg) {
570 assert(kreg == knoreg || kreg->is_valid(), "sanity");
571 int kreg_enc = kreg->raw_encoding();
572 assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
573 return kreg_enc;
574 }
575
576 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
577 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
578 }
579
580 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
581 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
582 }
583
584 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
585 assert((mod & 3) != 0b11, "forbidden");
586 int modrm = modrm_encoding(mod, dst_enc, src_enc);
587 emit_int8(modrm);
588 }
589
590 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
591 int disp) {
592 int modrm = modrm_encoding(mod, dst_enc, src_enc);
593 emit_int16(modrm, disp & 0xFF);
594 }
595
596 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
597 Address::ScaleFactor scale, int index_enc, int base_enc) {
598 int modrm = modrm_encoding(mod, dst_enc, src_enc);
599 int sib = sib_encoding(scale, index_enc, base_enc);
600 emit_int16(modrm, sib);
601 }
602
603 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
604 Address::ScaleFactor scale, int index_enc, int base_enc,
605 int disp) {
606 int modrm = modrm_encoding(mod, dst_enc, src_enc);
607 int sib = sib_encoding(scale, index_enc, base_enc);
608 emit_int24(modrm, sib, disp & 0xFF);
609 }
610
611 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
612 Address::ScaleFactor scale, int disp,
613 RelocationHolder const& rspec,
614 int post_addr_length) {
615 bool no_relocation = (rspec.type() == relocInfo::none);
616
617 if (is_valid_encoding(base_enc)) {
618 if (is_valid_encoding(index_enc)) {
619 assert(scale != Address::no_scale, "inconsistent address");
620 // [base + index*scale + disp]
621 if (disp == 0 && no_relocation && ((base_enc & 0x7) != 5)) {
622 // [base + index*scale]
623 // !(rbp | r13 | r21 | r29)
624 // [00 reg 100][ss index base]
625 emit_modrm_sib(0b00, reg_enc, 0b100,
626 scale, index_enc, base_enc);
627 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
628 // [base + index*scale + imm8]
629 // [01 reg 100][ss index base] imm8
630 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
631 scale, index_enc, base_enc,
632 disp);
633 } else {
634 // [base + index*scale + disp32]
635 // [10 reg 100][ss index base] disp32
636 emit_modrm_sib(0b10, reg_enc, 0b100,
637 scale, index_enc, base_enc);
638 emit_data(disp, rspec, disp32_operand);
639 }
640 } else if ((base_enc & 0x7) == 4) {
641 // rsp | r12 | r20 | r28
642 // [rsp + disp]
643 if (disp == 0 && no_relocation) {
644 // [rsp]
645 // [00 reg 100][00 100 100]
646 emit_modrm_sib(0b00, reg_enc, 0b100,
647 Address::times_1, 0b100, 0b100);
648 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
649 // [rsp + imm8]
650 // [01 reg 100][00 100 100] disp8
651 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
652 Address::times_1, 0b100, 0b100,
653 disp);
654 } else {
655 // [rsp + imm32]
656 // [10 reg 100][00 100 100] disp32
657 emit_modrm_sib(0b10, reg_enc, 0b100,
658 Address::times_1, 0b100, 0b100);
659 emit_data(disp, rspec, disp32_operand);
660 }
661 } else {
662 // [base + disp]
663 // !(rsp | r12 | r20 | r28) were handled above
664 assert(((base_enc & 0x7) != 4), "illegal addressing mode");
665 if (disp == 0 && no_relocation && ((base_enc & 0x7) != 5)) {
666 // [base]
667 // !(rbp | r13 | r21 | r29)
668 // [00 reg base]
669 emit_modrm(0, reg_enc, base_enc);
670 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
671 // [base + disp8]
672 // [01 reg base] disp8
673 emit_modrm_disp8(0b01, reg_enc, base_enc,
674 disp);
675 } else {
676 // [base + disp32]
677 // [10 reg base] disp32
678 emit_modrm(0b10, reg_enc, base_enc);
679 emit_data(disp, rspec, disp32_operand);
680 }
681 }
682 } else {
683 if (is_valid_encoding(index_enc)) {
684 assert(scale != Address::no_scale, "inconsistent address");
685 // base == noreg
686 // [index*scale + disp]
687 // [00 reg 100][ss index 101] disp32
688 emit_modrm_sib(0b00, reg_enc, 0b100,
689 scale, index_enc, 0b101 /* no base */);
690 emit_data(disp, rspec, disp32_operand);
691 } else if (!no_relocation) {
692 // base == noreg, index == noreg
693 // [disp] (64bit) RIP-RELATIVE (32bit) abs
694 // [00 reg 101] disp32
695
696 emit_modrm(0b00, reg_enc, 0b101 /* no base */);
697 // Note that the RIP-rel. correction applies to the generated
698 // disp field, but _not_ to the target address in the rspec.
699
700 // disp was created by converting the target address minus the pc
701 // at the start of the instruction. That needs more correction here.
702 // intptr_t disp = target - next_ip;
703 assert(inst_mark() != nullptr, "must be inside InstructionMark");
704 address next_ip = pc() + sizeof(int32_t) + post_addr_length;
705 int64_t adjusted = disp;
706 // Do rip-rel adjustment
707 adjusted -= (next_ip - inst_mark());
708 assert(is_simm32(adjusted),
709 "must be 32bit offset (RIP relative address)");
710 emit_data((int32_t) adjusted, rspec, disp32_operand);
711
712 } else {
713 // base == noreg, index == noreg, no_relocation == true
714 // 32bit never did this, did everything as the rip-rel/disp code above
715 // [disp] ABSOLUTE
716 // [00 reg 100][00 100 101] disp32
717 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
718 Address::times_1, 0b100, 0b101);
719 emit_data(disp, rspec, disp32_operand);
720 }
721 }
722 }
723
724 void Assembler::emit_operand(Register reg, Register base, Register index,
725 Address::ScaleFactor scale, int disp,
726 RelocationHolder const& rspec,
727 int post_addr_length) {
728 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
729 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
730 scale, disp, rspec, post_addr_length);
731
732 }
733 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
734 Address::ScaleFactor scale, int disp,
735 RelocationHolder const& rspec,
736 int post_addr_length) {
737 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
738 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
739 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
740 scale, disp, rspec, post_addr_length);
741 }
742
743 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
744 Address::ScaleFactor scale, int disp,
745 RelocationHolder const& rspec,
746 int post_addr_length) {
747 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
748 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
749 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
750 scale, disp, rspec, post_addr_length);
751 }
752
753 void Assembler::emit_operand(KRegister kreg, Address adr,
754 int post_addr_length) {
755 emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
756 adr._rspec,
757 post_addr_length);
758 }
759
760 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
761 Address::ScaleFactor scale, int disp,
762 RelocationHolder const& rspec,
763 int post_addr_length) {
764 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
765 emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
766 scale, disp, rspec, post_addr_length);
767 }
768
769 // Secret local extension to Assembler::WhichOperand:
770 #define end_pc_operand (_WhichOperand_limit)
771
772 address Assembler::locate_operand(address inst, WhichOperand which) {
773 // Decode the given instruction, and return the address of
774 // an embedded 32-bit operand word.
775
776 // If "which" is disp32_operand, selects the displacement portion
777 // of an effective address specifier.
778 // If "which" is imm64_operand, selects the trailing immediate constant.
779 // If "which" is call32_operand, selects the displacement of a call or jump.
780 // Caller is responsible for ensuring that there is such an operand,
781 // and that it is 32/64 bits wide.
782
783 // If "which" is end_pc_operand, find the end of the instruction.
784
785 address ip = inst;
786 bool is_64bit = false;
787
788 DEBUG_ONLY(bool has_disp32 = false);
789 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
790
791 again_after_prefix:
792 switch (0xFF & *ip++) {
793
794 // These convenience macros generate groups of "case" labels for the switch.
795 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
796 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
797 case (x)+4: case (x)+5: case (x)+6: case (x)+7
798 #define REP16(x) REP8((x)+0): \
799 case REP8((x)+8)
800
801 case CS_segment:
802 case SS_segment:
803 case DS_segment:
804 case ES_segment:
805 case FS_segment:
806 case GS_segment:
807 // Seems dubious
808 assert(false, "shouldn't have that prefix");
809 assert(ip == inst+1, "only one prefix allowed");
810 goto again_after_prefix;
811
812 case 0x67:
813 case REX:
814 case REX_B:
815 case REX_X:
816 case REX_XB:
817 case REX_R:
818 case REX_RB:
819 case REX_RX:
820 case REX_RXB:
821 goto again_after_prefix;
822
823 case REX2:
824 if ((0xFF & *ip++) & REX2BIT_W) {
825 is_64bit = true;
826 }
827 goto again_after_prefix;
828
829 case REX_W:
830 case REX_WB:
831 case REX_WX:
832 case REX_WXB:
833 case REX_WR:
834 case REX_WRB:
835 case REX_WRX:
836 case REX_WRXB:
837 is_64bit = true;
838 goto again_after_prefix;
839
840 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
841 case 0x88: // movb a, r
842 case 0x89: // movl a, r
843 case 0x8A: // movb r, a
844 case 0x8B: // movl r, a
845 case 0x8F: // popl a
846 DEBUG_ONLY(has_disp32 = true);
847 break;
848
849 case 0x68: // pushq #32
850 if (which == end_pc_operand) {
851 return ip + 4;
852 }
853 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
854 return ip; // not produced by emit_operand
855
856 case 0x66: // movw ... (size prefix)
857 again_after_size_prefix2:
858 switch (0xFF & *ip++) {
859 case REX:
860 case REX_B:
861 case REX_X:
862 case REX_XB:
863 case REX_R:
864 case REX_RB:
865 case REX_RX:
866 case REX_RXB:
867 case REX_W:
868 case REX_WB:
869 case REX_WX:
870 case REX_WXB:
871 case REX_WR:
872 case REX_WRB:
873 case REX_WRX:
874 case REX_WRXB:
875 goto again_after_size_prefix2;
876
877 case REX2:
878 if ((0xFF & *ip++) & REX2BIT_W) {
879 is_64bit = true;
880 }
881 goto again_after_size_prefix2;
882
883 case 0x8B: // movw r, a
884 case 0x89: // movw a, r
885 DEBUG_ONLY(has_disp32 = true);
886 break;
887 case 0xC7: // movw a, #16
888 DEBUG_ONLY(has_disp32 = true);
889 tail_size = 2; // the imm16
890 break;
891 case 0x0F: // several SSE/SSE2 variants
892 ip--; // reparse the 0x0F
893 goto again_after_prefix;
894 default:
895 ShouldNotReachHere();
896 }
897 break;
898
899 case REP8(0xB8): // movl/q r, #32/#64(oop?)
900 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
901 // these asserts are somewhat nonsensical
902 assert(((which == call32_operand || which == imm_operand) && is_64bit) ||
903 (which == narrow_oop_operand && !is_64bit),
904 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
905 return ip;
906
907 case 0x69: // imul r, a, #32
908 case 0xC7: // movl a, #32(oop?)
909 tail_size = 4;
910 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
911 break;
912
913 case 0x0F: // movx..., etc.
914 switch (0xFF & *ip++) {
915 case 0x3A: // pcmpestri
916 tail_size = 1;
917 case 0x38: // ptest, pmovzxbw
918 ip++; // skip opcode
919 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
920 break;
921
922 case 0x70: // pshufd r, r/a, #8
923 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
924 case 0x73: // psrldq r, #8
925 tail_size = 1;
926 break;
927
928 case 0x10: // movups
929 case 0x11: // movups
930 case 0x12: // movlps
931 case 0x28: // movaps
932 case 0x29: // movaps
933 case 0x2E: // ucomiss
934 case 0x2F: // comiss
935 case 0x54: // andps
936 case 0x55: // andnps
937 case 0x56: // orps
938 case 0x57: // xorps
939 case 0x58: // addpd
940 case 0x59: // mulpd
941 case 0x6E: // movd
942 case 0x7E: // movd
943 case 0x6F: // movdq
944 case 0x7F: // movdq
945 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
946 case 0xD6: // movq
947 case 0xFE: // paddd
948 DEBUG_ONLY(has_disp32 = true);
949 break;
950
951 case 0xAD: // shrd r, a, %cl
952 case 0xAF: // imul r, a
953 case 0xBE: // movsbl r, a (movsxb)
954 case 0xBF: // movswl r, a (movsxw)
955 case 0xB6: // movzbl r, a (movzxb)
956 case 0xB7: // movzwl r, a (movzxw)
957 case REP16(0x40): // cmovl cc, r, a
958 case 0xB0: // cmpxchgb
959 case 0xB1: // cmpxchg
960 case 0xC1: // xaddl
961 case 0xC7: // cmpxchg8
962 case REP16(0x90): // setcc a
963 DEBUG_ONLY(has_disp32 = true);
964 // fall out of the switch to decode the address
965 break;
966
967 case 0xC4: // pinsrw r, a, #8
968 DEBUG_ONLY(has_disp32 = true);
969 case 0xC5: // pextrw r, r, #8
970 tail_size = 1; // the imm8
971 break;
972
973 case 0xAC: // shrd r, a, #8
974 DEBUG_ONLY(has_disp32 = true);
975 tail_size = 1; // the imm8
976 break;
977
978 case REP16(0x80): // jcc rdisp32
979 if (which == end_pc_operand) return ip + 4;
980 assert(which == call32_operand, "jcc has no disp32 or imm");
981 return ip;
982 default:
983 fatal("not handled: 0x0F%2X", 0xFF & *(ip-1));
984 }
985 break;
986
987 case 0x81: // addl a, #32; addl r, #32
988 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
989 // on 32bit in the case of cmpl, the imm might be an oop
990 tail_size = 4;
991 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
992 break;
993
994 case 0x83: // addl a, #8; addl r, #8
995 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
996 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
997 tail_size = 1;
998 break;
999
1000 case 0x15: // adc rax, #32
1001 case 0x05: // add rax, #32
1002 case 0x25: // and rax, #32
1003 case 0x3D: // cmp rax, #32
1004 case 0x0D: // or rax, #32
1005 case 0x1D: // sbb rax, #32
1006 case 0x2D: // sub rax, #32
1007 case 0x35: // xor rax, #32
1008 return which == end_pc_operand ? ip + 4 : ip;
1009
1010 case 0x9B:
1011 switch (0xFF & *ip++) {
1012 case 0xD9: // fnstcw a
1013 DEBUG_ONLY(has_disp32 = true);
1014 break;
1015 default:
1016 ShouldNotReachHere();
1017 }
1018 break;
1019
1020 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
1021 case REP4(0x10): // adc...
1022 case REP4(0x20): // and...
1023 case REP4(0x30): // xor...
1024 case REP4(0x08): // or...
1025 case REP4(0x18): // sbb...
1026 case REP4(0x28): // sub...
1027 case 0xF7: // mull a
1028 case 0x8D: // lea r, a
1029 case 0x87: // xchg r, a
1030 case REP4(0x38): // cmp...
1031 case 0x85: // test r, a
1032 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
1033 break;
1034
1035 case 0xA8: // testb rax, #8
1036 return which == end_pc_operand ? ip + 1 : ip;
1037 case 0xA9: // testl/testq rax, #32
1038 return which == end_pc_operand ? ip + 4 : ip;
1039
1040 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
1041 case 0xC6: // movb a, #8
1042 case 0x80: // cmpb a, #8
1043 case 0x6B: // imul r, a, #8
1044 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
1045 tail_size = 1; // the imm8
1046 break;
1047
1048 case 0xC4: // VEX_3bytes
1049 case 0xC5: // VEX_2bytes
1050 assert((UseAVX > 0), "shouldn't have VEX prefix");
1051 assert(ip == inst+1, "no prefixes allowed");
1052 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
1053 // but they have prefix 0x0F and processed when 0x0F processed above.
1054 //
1055 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
1056 // instructions (these instructions are not supported in 64-bit mode).
1057 // To distinguish them bits [7:6] are set in the VEX second byte since
1058 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
1059 // those VEX bits REX and vvvv bits are inverted.
1060 //
1061 // Fortunately C2 doesn't generate these instructions so we don't need
1062 // to check for them in product version.
1063
1064 // Check second byte
1065 int vex_opcode;
1066 // First byte
1067 if ((0xFF & *inst) == VEX_3bytes) {
1068 vex_opcode = VEX_OPCODE_MASK & *ip;
1069 ip++; // third byte
1070 is_64bit = ((VEX_W & *ip) == VEX_W);
1071 } else {
1072 vex_opcode = VEX_OPCODE_0F;
1073 }
1074 ip++; // opcode
1075 // To find the end of instruction (which == end_pc_operand).
1076 switch (vex_opcode) {
1077 case VEX_OPCODE_0F:
1078 switch (0xFF & *ip) {
1079 case 0x70: // pshufd r, r/a, #8
1080 case 0x71: // ps[rl|ra|ll]w r, #8
1081 case 0x72: // ps[rl|ra|ll]d r, #8
1082 case 0x73: // ps[rl|ra|ll]q r, #8
1083 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
1084 case 0xC4: // pinsrw r, r, r/a, #8
1085 case 0xC5: // pextrw r/a, r, #8
1086 case 0xC6: // shufp[s|d] r, r, r/a, #8
1087 tail_size = 1; // the imm8
1088 break;
1089 }
1090 break;
1091 case VEX_OPCODE_0F_3A:
1092 tail_size = 1;
1093 break;
1094 }
1095 ip++; // skip opcode
1096 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
1097 break;
1098
1099 case 0x62: // EVEX_4bytes
1100 assert(VM_Version::cpu_supports_evex(), "shouldn't have EVEX prefix");
1101 assert(ip == inst+1, "no prefixes allowed");
1102 // no EVEX collisions, all instructions that have 0x62 opcodes
1103 // have EVEX versions and are subopcodes of 0x66
1104 ip++; // skip P0 and examine W in P1
1105 is_64bit = ((VEX_W & *ip) == VEX_W);
1106 ip++; // move to P2
1107 ip++; // skip P2, move to opcode
1108 // To find the end of instruction (which == end_pc_operand).
1109 switch (0xFF & *ip) {
1110 case 0x22: // pinsrd r, r/a, #8
1111 case 0x61: // pcmpestri r, r/a, #8
1112 case 0x70: // pshufd r, r/a, #8
1113 case 0x73: // psrldq r, #8
1114 case 0x1f: // evpcmpd/evpcmpq
1115 case 0x3f: // evpcmpb/evpcmpw
1116 tail_size = 1; // the imm8
1117 break;
1118 default:
1119 break;
1120 }
1121 ip++; // skip opcode
1122 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
1123 break;
1124
1125 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
1126 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
1127 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
1128 case 0xDD: // fld_d a; fst_d a; fstp_d a
1129 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
1130 case 0xDF: // fild_d a; fistp_d a
1131 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
1132 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
1133 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
1134 DEBUG_ONLY(has_disp32 = true);
1135 break;
1136
1137 case 0xE8: // call rdisp32
1138 case 0xE9: // jmp rdisp32
1139 if (which == end_pc_operand) return ip + 4;
1140 assert(which == call32_operand, "call has no disp32 or imm");
1141 return ip;
1142
1143 case 0xF0: // Lock
1144 goto again_after_prefix;
1145
1146 case 0xF3: // For SSE
1147 case 0xF2: // For SSE2
1148 switch (0xFF & *ip++) {
1149 case REX:
1150 case REX_B:
1151 case REX_X:
1152 case REX_XB:
1153 case REX_R:
1154 case REX_RB:
1155 case REX_RX:
1156 case REX_RXB:
1157 case REX_W:
1158 case REX_WB:
1159 case REX_WX:
1160 case REX_WXB:
1161 case REX_WR:
1162 case REX_WRB:
1163 case REX_WRX:
1164 case REX_WRXB:
1165 case REX2:
1166 ip++;
1167 // fall-through
1168 default:
1169 ip++;
1170 }
1171 DEBUG_ONLY(has_disp32 = true); // has both kinds of operands!
1172 break;
1173
1174 default:
1175 ShouldNotReachHere();
1176
1177 #undef REP8
1178 #undef REP16
1179 }
1180
1181 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1182 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1183 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1184
1185 // parse the output of emit_operand
1186 int op2 = 0xFF & *ip++;
1187 int base = op2 & 0x07;
1188 int op3 = -1;
1189 const int b100 = 4;
1190 const int b101 = 5;
1191 if (base == b100 && (op2 >> 6) != 3) {
1192 op3 = 0xFF & *ip++;
1193 base = op3 & 0x07; // refetch the base
1194 }
1195 // now ip points at the disp (if any)
1196
1197 switch (op2 >> 6) {
1198 case 0:
1199 // [00 reg 100][ss index base]
1200 // [00 reg 100][00 100 esp]
1201 // [00 reg base]
1202 // [00 reg 100][ss index 101][disp32]
1203 // [00 reg 101] [disp32]
1204
1205 if (base == b101) {
1206 if (which == disp32_operand)
1207 return ip; // caller wants the disp32
1208 ip += 4; // skip the disp32
1209 }
1210 break;
1211
1212 case 1:
1213 // [01 reg 100][ss index base][disp8]
1214 // [01 reg 100][00 100 esp][disp8]
1215 // [01 reg base] [disp8]
1216 ip += 1; // skip the disp8
1217 break;
1218
1219 case 2:
1220 // [10 reg 100][ss index base][disp32]
1221 // [10 reg 100][00 100 esp][disp32]
1222 // [10 reg base] [disp32]
1223 if (which == disp32_operand)
1224 return ip; // caller wants the disp32
1225 ip += 4; // skip the disp32
1226 break;
1227
1228 case 3:
1229 // [11 reg base] (not a memory addressing mode)
1230 break;
1231 }
1232
1233 if (which == end_pc_operand) {
1234 return ip + tail_size;
1235 }
1236
1237 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1238 return ip;
1239 }
1240
1241 address Assembler::locate_next_instruction(address inst) {
1242 // Secretly share code with locate_operand:
1243 return locate_operand(inst, end_pc_operand);
1244 }
1245
1246
1247 #ifdef ASSERT
1248 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1249 address inst = inst_mark();
1250 assert(inst != nullptr && inst < pc(), "must point to beginning of instruction");
1251 address opnd;
1252
1253 Relocation* r = rspec.reloc();
1254 if (r->type() == relocInfo::none) {
1255 return;
1256 } else if (r->is_call() || format == call32_operand) {
1257 // assert(format == imm32_operand, "cannot specify a nonzero format");
1258 opnd = locate_operand(inst, call32_operand);
1259 } else if (r->is_data()) {
1260 assert(format == imm_operand || format == disp32_operand || format == narrow_oop_operand, "format ok");
1261 opnd = locate_operand(inst, (WhichOperand)format);
1262 } else {
1263 assert(format == imm_operand, "cannot specify a format");
1264 return;
1265 }
1266 assert(opnd == pc(), "must put operand where relocs can find it");
1267 }
1268 #endif // ASSERT
1269
1270 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
1271 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1272 }
1273
1274 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
1275 if (adr.isxmmindex()) {
1276 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
1277 } else {
1278 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1279 }
1280 }
1281
1282 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int byte2, int ocp_and_encoding, int byte3) {
1283 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8;
1284 if (opcode_prefix != 0) {
1285 emit_int32(opcode_prefix, (unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF), byte3);
1286 } else {
1287 emit_int24((unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF), byte3);
1288 }
1289 }
1290
1291 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int byte2, int ocp_and_encoding) {
1292 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8;
1293 if (opcode_prefix != 0) {
1294 emit_int24(opcode_prefix, (unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF));
1295 } else {
1296 emit_int16((unsigned char)byte1, byte2 | (ocp_and_encoding & 0xFF));
1297 }
1298 }
1299
1300 void Assembler::emit_opcode_prefix_and_encoding(int byte1, int ocp_and_encoding) {
1301 int opcode_prefix = (ocp_and_encoding & 0xFF00) >> 8;
1302 if (opcode_prefix != 0) {
1303 emit_int16(opcode_prefix, (unsigned char)byte1 | (ocp_and_encoding & 0xFF));
1304 } else {
1305 emit_int8((unsigned char)byte1 | (ocp_and_encoding & 0xFF));
1306 }
1307 }
1308
1309 // Now the Assembler instructions (identical for 32/64 bits)
1310
1311 void Assembler::adcl(Address dst, int32_t imm32) {
1312 InstructionMark im(this);
1313 prefix(dst);
1314 emit_arith_operand(0x81, rdx, dst, imm32);
1315 }
1316
1317 void Assembler::adcl(Address dst, Register src) {
1318 InstructionMark im(this);
1319 prefix(dst, src);
1320 emit_int8(0x11);
1321 emit_operand(src, dst, 0);
1322 }
1323
1324 void Assembler::adcl(Register dst, int32_t imm32) {
1325 prefix(dst);
1326 emit_arith(0x81, 0xD0, dst, imm32);
1327 }
1328
1329 void Assembler::adcl(Register dst, Address src) {
1330 InstructionMark im(this);
1331 prefix(src, dst);
1332 emit_int8(0x13);
1333 emit_operand(dst, src, 0);
1334 }
1335
1336 void Assembler::adcl(Register dst, Register src) {
1337 (void) prefix_and_encode(dst->encoding(), src->encoding());
1338 emit_arith(0x13, 0xC0, dst, src);
1339 }
1340
1341 void Assembler::addl(Address dst, int32_t imm32) {
1342 InstructionMark im(this);
1343 prefix(dst);
1344 emit_arith_operand(0x81, rax, dst, imm32);
1345 }
1346
1347 void Assembler::eaddl(Register dst, Address src, int32_t imm32, bool no_flags) {
1348 InstructionMark im(this);
1349 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1350 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1351 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
1352 emit_arith_operand(0x81, rax, src, imm32);
1353 }
1354
1355 void Assembler::addb(Address dst, int imm8) {
1356 InstructionMark im(this);
1357 prefix(dst);
1358 emit_int8((unsigned char)0x80);
1359 emit_operand(rax, dst, 1);
1360 emit_int8(imm8);
1361 }
1362
1363 void Assembler::addb(Address dst, Register src) {
1364 InstructionMark im(this);
1365 prefix(dst, src);
1366 emit_int8(0x00);
1367 emit_operand(src, dst, 0);
1368 }
1369
1370 void Assembler::addb(Register dst, int imm8) {
1371 (void) prefix_and_encode(dst->encoding(), true);
1372 emit_arith_b(0x80, 0xC0, dst, imm8);
1373 }
1374
1375 void Assembler::addw(Address dst, int imm16) {
1376 InstructionMark im(this);
1377 emit_int8(0x66);
1378 prefix(dst);
1379 emit_int8((unsigned char)0x81);
1380 emit_operand(rax, dst, 2);
1381 emit_int16(imm16);
1382 }
1383
1384 void Assembler::addw(Address dst, Register src) {
1385 InstructionMark im(this);
1386 emit_int8(0x66);
1387 prefix(dst, src);
1388 emit_int8(0x01);
1389 emit_operand(src, dst, 0);
1390 }
1391
1392 void Assembler::addl(Address dst, Register src) {
1393 InstructionMark im(this);
1394 prefix(dst, src);
1395 emit_int8(0x01);
1396 emit_operand(src, dst, 0);
1397 }
1398
1399 void Assembler::eaddl(Register dst, Address src1, Register src2, bool no_flags) {
1400 InstructionMark im(this);
1401 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
1402 }
1403
1404 void Assembler::addl(Register dst, int32_t imm32) {
1405 prefix(dst);
1406 emit_arith(0x81, 0xC0, dst, imm32);
1407 }
1408
1409 void Assembler::eaddl(Register dst, Register src, int32_t imm32, bool no_flags) {
1410 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xC0, no_flags);
1411 }
1412
1413 void Assembler::addl(Register dst, Address src) {
1414 InstructionMark im(this);
1415 prefix(src, dst);
1416 emit_int8(0x03);
1417 emit_operand(dst, src, 0);
1418 }
1419
1420 void Assembler::eaddl(Register dst, Register src1, Address src2, bool no_flags) {
1421 InstructionMark im(this);
1422 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, no_flags);
1423 }
1424
1425 void Assembler::addl(Register dst, Register src) {
1426 (void) prefix_and_encode(dst->encoding(), src->encoding());
1427 emit_arith(0x03, 0xC0, dst, src);
1428 }
1429
1430 void Assembler::eaddl(Register dst, Register src1, Register src2, bool no_flags) {
1431 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
1432 }
1433
1434 void Assembler::addr_nop_4() {
1435 assert(UseAddressNop, "no CPU support");
1436 // 4 bytes: NOP DWORD PTR [EAX+0]
1437 emit_int32(0x0F,
1438 0x1F,
1439 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1440 0); // 8-bits offset (1 byte)
1441 }
1442
1443 void Assembler::addr_nop_5() {
1444 assert(UseAddressNop, "no CPU support");
1445 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1446 emit_int32(0x0F,
1447 0x1F,
1448 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1449 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1450 emit_int8(0); // 8-bits offset (1 byte)
1451 }
1452
1453 void Assembler::addr_nop_7() {
1454 assert(UseAddressNop, "no CPU support");
1455 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1456 emit_int24(0x0F,
1457 0x1F,
1458 (unsigned char)0x80);
1459 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1460 emit_int32(0); // 32-bits offset (4 bytes)
1461 }
1462
1463 void Assembler::addr_nop_8() {
1464 assert(UseAddressNop, "no CPU support");
1465 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1466 emit_int32(0x0F,
1467 0x1F,
1468 (unsigned char)0x84,
1469 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1470 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1471 emit_int32(0); // 32-bits offset (4 bytes)
1472 }
1473
1474 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1475 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1476 attributes.set_rex_vex_w_reverted();
1477 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1478 emit_int16(0x58, (0xC0 | encode));
1479 }
1480
1481 void Assembler::addsd(XMMRegister dst, Address src) {
1482 InstructionMark im(this);
1483 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1484 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1485 attributes.set_rex_vex_w_reverted();
1486 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1487 emit_int8(0x58);
1488 emit_operand(dst, src, 0);
1489 }
1490
1491 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1492 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1493 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1494 emit_int16(0x58, (0xC0 | encode));
1495 }
1496
1497 void Assembler::addss(XMMRegister dst, Address src) {
1498 InstructionMark im(this);
1499 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1500 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1501 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1502 emit_int8(0x58);
1503 emit_operand(dst, src, 0);
1504 }
1505
1506 void Assembler::aesdec(XMMRegister dst, Address src) {
1507 assert(VM_Version::supports_aes(), "");
1508 InstructionMark im(this);
1509 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1510 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1511 emit_int8((unsigned char)0xDE);
1512 emit_operand(dst, src, 0);
1513 }
1514
1515 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1516 assert(VM_Version::supports_aes(), "");
1517 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1518 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1519 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1520 }
1521
1522 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1523 assert(VM_Version::supports_avx512_vaes(), "");
1524 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1525 attributes.set_is_evex_instruction();
1526 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1527 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1528 }
1529
1530
1531 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1532 assert(VM_Version::supports_aes(), "");
1533 InstructionMark im(this);
1534 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1535 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1536 emit_int8((unsigned char)0xDF);
1537 emit_operand(dst, src, 0);
1538 }
1539
1540 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1541 assert(VM_Version::supports_aes(), "");
1542 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1543 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1544 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1545 }
1546
1547 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1548 assert(VM_Version::supports_avx512_vaes(), "");
1549 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1550 attributes.set_is_evex_instruction();
1551 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1552 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1553 }
1554
1555 void Assembler::aesenc(XMMRegister dst, Address src) {
1556 assert(VM_Version::supports_aes(), "");
1557 InstructionMark im(this);
1558 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1559 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1560 emit_int8((unsigned char)0xDC);
1561 emit_operand(dst, src, 0);
1562 }
1563
1564 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1565 assert(VM_Version::supports_aes(), "");
1566 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1567 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1568 emit_int16((unsigned char)0xDC, 0xC0 | encode);
1569 }
1570
1571 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1572 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1574 attributes.set_is_evex_instruction();
1575 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1576 emit_int16((unsigned char)0xDC, (0xC0 | encode));
1577 }
1578
1579 void Assembler::aesenclast(XMMRegister dst, Address src) {
1580 assert(VM_Version::supports_aes(), "");
1581 InstructionMark im(this);
1582 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1583 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1584 emit_int8((unsigned char)0xDD);
1585 emit_operand(dst, src, 0);
1586 }
1587
1588 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1589 assert(VM_Version::supports_aes(), "");
1590 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1591 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1592 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1593 }
1594
1595 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1596 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1597 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1598 attributes.set_is_evex_instruction();
1599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1600 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1601 }
1602
1603 void Assembler::andb(Address dst, Register src) {
1604 InstructionMark im(this);
1605 prefix(dst, src, true);
1606 emit_int8(0x20);
1607 emit_operand(src, dst, 0);
1608 }
1609
1610 void Assembler::andl(Address dst, int32_t imm32) {
1611 InstructionMark im(this);
1612 prefix(dst);
1613 emit_arith_operand(0x81, as_Register(4), dst, imm32);
1614 }
1615
1616 void Assembler::eandl(Register dst, Address src, int32_t imm32, bool no_flags) {
1617 InstructionMark im(this);
1618 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1619 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1620 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
1621 emit_arith_operand(0x81, rsp, src, imm32);
1622 }
1623
1624 void Assembler::andl(Register dst, int32_t imm32) {
1625 prefix(dst);
1626 emit_arith(0x81, 0xE0, dst, imm32);
1627 }
1628
1629 void Assembler::eandl(Register dst, Register src, int32_t imm32, bool no_flags) {
1630 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xE0, no_flags);
1631 }
1632
1633 void Assembler::andl(Address dst, Register src) {
1634 InstructionMark im(this);
1635 prefix(dst, src);
1636 emit_int8(0x21);
1637 emit_operand(src, dst, 0);
1638 }
1639
1640 void Assembler::andl(Register dst, Address src) {
1641 InstructionMark im(this);
1642 prefix(src, dst);
1643 emit_int8(0x23);
1644 emit_operand(dst, src, 0);
1645 }
1646
1647 void Assembler::eandl(Register dst, Register src1, Address src2, bool no_flags) {
1648 InstructionMark im(this);
1649 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, no_flags);
1650 }
1651
1652 void Assembler::eandl(Register dst, Address src1, Register src2, bool no_flags) {
1653 InstructionMark im(this);
1654 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
1655 }
1656
1657 void Assembler::andl(Register dst, Register src) {
1658 (void) prefix_and_encode(dst->encoding(), src->encoding());
1659 emit_arith(0x23, 0xC0, dst, src);
1660 }
1661
1662 void Assembler::eandl(Register dst, Register src1, Register src2, bool no_flags) {
1663 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
1664 }
1665
1666 void Assembler::andnl(Register dst, Register src1, Register src2) {
1667 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1668 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1669 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
1670 emit_int16((unsigned char)0xF2, (0xC0 | encode));
1671 }
1672
1673 void Assembler::andnl(Register dst, Register src1, Address src2) {
1674 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1675 InstructionMark im(this);
1676 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1677 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1678 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1679 emit_int8((unsigned char)0xF2);
1680 emit_operand(dst, src2, 0);
1681 }
1682
1683 void Assembler::bsfl(Register dst, Register src) {
1684 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
1685 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
1686 }
1687
1688 void Assembler::bsrl(Register dst, Register src) {
1689 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
1690 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode);
1691 }
1692
1693 void Assembler::bswapl(Register reg) { // bswap
1694 int encode = prefix_and_encode(reg->encoding(), false, true /* is_map1 */);
1695 emit_opcode_prefix_and_encoding((unsigned char)0xC8, encode);
1696 }
1697
1698 void Assembler::blsil(Register dst, Register src) {
1699 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1700 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1701 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
1702 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1703 }
1704
1705 void Assembler::blsil(Register dst, Address src) {
1706 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1707 InstructionMark im(this);
1708 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1709 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1710 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1711 emit_int8((unsigned char)0xF3);
1712 emit_operand(rbx, src, 0);
1713 }
1714
1715 void Assembler::blsmskl(Register dst, Register src) {
1716 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1717 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1718 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
1719 emit_int16((unsigned char)0xF3,
1720 0xC0 | encode);
1721 }
1722
1723 void Assembler::blsmskl(Register dst, Address src) {
1724 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1725 InstructionMark im(this);
1726 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1727 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1728 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1729 emit_int8((unsigned char)0xF3);
1730 emit_operand(rdx, src, 0);
1731 }
1732
1733 void Assembler::blsrl(Register dst, Register src) {
1734 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1735 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1736 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
1737 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1738 }
1739
1740 void Assembler::blsrl(Register dst, Address src) {
1741 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1742 InstructionMark im(this);
1743 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1744 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
1745 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1746 emit_int8((unsigned char)0xF3);
1747 emit_operand(rcx, src, 0);
1748 }
1749
1750 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1751 if (L.is_bound()) {
1752 const int long_size = 5;
1753 int offs = (int)( target(L) - pc() );
1754 assert(offs <= 0, "assembler error");
1755 InstructionMark im(this);
1756 // 1110 1000 #32-bit disp
1757 emit_int8((unsigned char)0xE8);
1758 emit_data(offs - long_size, rtype, disp32_operand);
1759 } else {
1760 InstructionMark im(this);
1761 // 1110 1000 #32-bit disp
1762 L.add_patch_at(code(), locator());
1763
1764 emit_int8((unsigned char)0xE8);
1765 emit_data(int(0), rtype, disp32_operand);
1766 }
1767 }
1768
1769 void Assembler::call(Register dst) {
1770 int encode = prefix_and_encode(dst->encoding());
1771 emit_int16((unsigned char)0xFF, (0xD0 | encode));
1772 }
1773
1774
1775 void Assembler::call(Address adr) {
1776 assert(!adr._rspec.reloc()->is_data(), "should not use ExternalAddress for call");
1777 InstructionMark im(this);
1778 prefix(adr);
1779 emit_int8((unsigned char)0xFF);
1780 emit_operand(rdx, adr, 0);
1781 }
1782
1783 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1784 InstructionMark im(this);
1785 emit_int8((unsigned char)0xE8);
1786 intptr_t disp = entry - (pc() + sizeof(int32_t));
1787 // Entry is null in case of a scratch emit.
1788 assert(entry == nullptr || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1789 // Technically, should use call32_operand, but this format is
1790 // implied by the fact that we're emitting a call instruction.
1791
1792 emit_data((int) disp, rspec, disp32_operand);
1793 }
1794
1795 void Assembler::cdql() {
1796 emit_int8((unsigned char)0x99);
1797 }
1798
1799 void Assembler::cld() {
1800 emit_int8((unsigned char)0xFC);
1801 }
1802
1803 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1804 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
1805 emit_opcode_prefix_and_encoding(0x40 | cc, 0xC0, encode);
1806 }
1807
1808 void Assembler::ecmovl(Condition cc, Register dst, Register src1, Register src2) {
1809 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x40 | cc, false /* no_flags */, true /* is_map1 */, true /* swap */);
1810 }
1811
1812 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1813 InstructionMark im(this);
1814 prefix(src, dst, false, true /* is_map1 */);
1815 emit_int8((0x40 | cc));
1816 emit_operand(dst, src, 0);
1817 }
1818
1819 void Assembler::ecmovl(Condition cc, Register dst, Register src1, Address src2) {
1820 InstructionMark im(this);
1821 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, (0x40 | cc) , false /* no_flags */, true /* is_map1 */);
1822 }
1823
1824 void Assembler::cmpb(Address dst, Register reg) {
1825 assert(reg->has_byte_register(), "must have byte register");
1826 InstructionMark im(this);
1827 prefix(dst, reg, true);
1828 emit_int8((unsigned char)0x38);
1829 emit_operand(reg, dst, 0);
1830 }
1831
1832 void Assembler::cmpb(Register reg, Address dst) {
1833 assert(reg->has_byte_register(), "must have byte register");
1834 InstructionMark im(this);
1835 prefix(dst, reg, true);
1836 emit_int8((unsigned char)0x3a);
1837 emit_operand(reg, dst, 0);
1838 }
1839
1840 void Assembler::cmpb(Address dst, int imm8) {
1841 InstructionMark im(this);
1842 prefix(dst);
1843 emit_int8((unsigned char)0x80);
1844 emit_operand(rdi, dst, 1);
1845 emit_int8(imm8);
1846 }
1847
1848 void Assembler::cmpb(Register dst, int imm8) {
1849 prefix(dst);
1850 emit_arith_b(0x80, 0xF8, dst, imm8);
1851 }
1852
1853 void Assembler::cmpl(Address dst, int32_t imm32) {
1854 InstructionMark im(this);
1855 prefix(dst);
1856 emit_arith_operand(0x81, as_Register(7), dst, imm32);
1857 }
1858
1859 void Assembler::cmpl(Register dst, int32_t imm32) {
1860 prefix(dst);
1861 emit_arith(0x81, 0xF8, dst, imm32);
1862 }
1863
1864 void Assembler::cmpl(Register dst, Register src) {
1865 (void) prefix_and_encode(dst->encoding(), src->encoding());
1866 emit_arith(0x3B, 0xC0, dst, src);
1867 }
1868
1869 void Assembler::cmpl(Register dst, Address src) {
1870 InstructionMark im(this);
1871 prefix(src, dst);
1872 emit_int8(0x3B);
1873 emit_operand(dst, src, 0);
1874 }
1875
1876 void Assembler::cmpl(Address dst, Register reg) {
1877 InstructionMark im(this);
1878 prefix(dst, reg);
1879 emit_int8(0x39);
1880 emit_operand(reg, dst, 0);
1881 }
1882
1883 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
1884 InstructionMark im(this);
1885 prefix(dst);
1886 emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
1887 }
1888
1889 void Assembler::cmpw(Address dst, int imm16) {
1890 InstructionMark im(this);
1891 emit_int8(0x66);
1892 prefix(dst);
1893 emit_int8((unsigned char)0x81);
1894 emit_operand(rdi, dst, 2);
1895 emit_int16(imm16);
1896 }
1897
1898 void Assembler::cmpw(Address dst, Register reg) {
1899 InstructionMark im(this);
1900 emit_int8(0x66);
1901 prefix(dst, reg);
1902 emit_int8((unsigned char)0x39);
1903 emit_operand(reg, dst, 0);
1904 }
1905
1906 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1907 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1908 // The ZF is set if the compared values were equal, and cleared otherwise.
1909 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1910 InstructionMark im(this);
1911 prefix(adr, reg, false, true /* is_map1 */);
1912 emit_int8((unsigned char)0xB1);
1913 emit_operand(reg, adr, 0);
1914 }
1915
1916 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
1917 InstructionMark im(this);
1918 size_prefix();
1919 prefix(adr, reg, false, true /* is_map1 */);
1920 emit_int8((unsigned char)0xB1);
1921 emit_operand(reg, adr, 0);
1922 }
1923
1924 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1925 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1926 // The ZF is set if the compared values were equal, and cleared otherwise.
1927 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1928 InstructionMark im(this);
1929 prefix(adr, reg, true, true /* is_map1 */);
1930 emit_int8((unsigned char)0xB0);
1931 emit_operand(reg, adr, 0);
1932 }
1933
1934 void Assembler::comisd(XMMRegister dst, Address src) {
1935 // NOTE: dbx seems to decode this as comiss even though the
1936 // 0x66 is there. Strangely ucomisd comes out correct
1937 InstructionMark im(this);
1938 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1939 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1940 attributes.set_rex_vex_w_reverted();
1941 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1942 emit_int8(0x2F);
1943 emit_operand(dst, src, 0);
1944 }
1945
1946 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1947 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1948 attributes.set_rex_vex_w_reverted();
1949 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1950 emit_int16(0x2F, (0xC0 | encode));
1951 }
1952
1953 void Assembler::comiss(XMMRegister dst, Address src) {
1954 InstructionMark im(this);
1955 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1956 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1957 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1958 emit_int8(0x2F);
1959 emit_operand(dst, src, 0);
1960 }
1961
1962 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1964 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1965 emit_int16(0x2F, (0xC0 | encode));
1966 }
1967
1968 void Assembler::cpuid() {
1969 emit_int16(0x0F, (unsigned char)0xA2);
1970 }
1971
1972 void Assembler::serialize() {
1973 assert(VM_Version::supports_serialize(), "");
1974 emit_int24(0x0F, 0x01, 0xE8);
1975 }
1976
1977 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1978 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1979 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1980 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1981 //
1982 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1983 //
1984 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1985 //
1986 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1987 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1988 assert(VM_Version::supports_sse4_2(), "");
1989 if (needs_eevex(crc, v)) {
1990 InstructionAttr attributes(AVX_128bit, /* rex_w */ sizeInBytes == 8, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1991 int encode = vex_prefix_and_encode(crc->encoding(), 0, v->encoding(), sizeInBytes == 2 ? VEX_SIMD_66 : VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true);
1992 emit_int16(sizeInBytes == 1 ? (unsigned char)0xF0 : (unsigned char)0xF1, (0xC0 | encode));
1993 } else {
1994 int8_t w = 0x01;
1995 Prefix p = Prefix_EMPTY;
1996
1997 emit_int8((unsigned char)0xF2);
1998 switch (sizeInBytes) {
1999 case 1:
2000 w = 0;
2001 break;
2002 case 2:
2003 case 4:
2004 break;
2005 case 8:
2006 // Note:
2007 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
2008 //
2009 // Page B - 72 Vol. 2C says
2010 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
2011 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
2012 // F0!!!
2013 // while 3 - 208 Vol. 2A
2014 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
2015 //
2016 // the 0 on a last bit is reserved for a different flavor of this instruction :
2017 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
2018 p = REX_W;
2019 break;
2020 default:
2021 assert(0, "Unsupported value for a sizeInBytes argument");
2022 break;
2023 }
2024 prefix(crc, v, p);
2025 emit_int32(0x0F,
2026 0x38,
2027 0xF0 | w,
2028 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
2029 }
2030 }
2031
2032 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
2033 assert(VM_Version::supports_sse4_2(), "");
2034 InstructionMark im(this);
2035 if (needs_eevex(crc, adr.base(), adr.index())) {
2036 InstructionAttr attributes(AVX_128bit, /* vex_w */ sizeInBytes == 8, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2037 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
2038 vex_prefix(adr, 0, crc->encoding(), sizeInBytes == 2 ? VEX_SIMD_66 : VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes);
2039 emit_int8(sizeInBytes == 1 ? (unsigned char)0xF0 : (unsigned char)0xF1);
2040 emit_operand(crc, adr, 0);
2041 } else {
2042 int8_t w = 0x01;
2043 Prefix p = Prefix_EMPTY;
2044
2045 emit_int8((uint8_t)0xF2);
2046 switch (sizeInBytes) {
2047 case 1:
2048 w = 0;
2049 break;
2050 case 2:
2051 case 4:
2052 break;
2053 case 8:
2054 p = REX_W;
2055 break;
2056 default:
2057 assert(0, "Unsupported value for a sizeInBytes argument");
2058 break;
2059 }
2060 prefix(crc, adr, p);
2061 emit_int24(0x0F, 0x38, (0xF0 | w));
2062 emit_operand(crc, adr, 0);
2063 }
2064 }
2065
2066 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
2067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2068 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2069 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2070 }
2071
2072 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2073 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2074 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2075 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2076 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2077 }
2078
2079 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
2080 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
2081 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
2082 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2083 emit_int24(0x1D, (0xC0 | encode), imm8);
2084 }
2085
2086 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
2087 assert(VM_Version::supports_evex(), "");
2088 InstructionMark im(this);
2089 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
2090 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
2091 attributes.reset_is_clear_context();
2092 attributes.set_embedded_opmask_register_specifier(mask);
2093 attributes.set_is_evex_instruction();
2094 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2095 emit_int8(0x1D);
2096 emit_operand(src, dst, 1);
2097 emit_int8(imm8);
2098 }
2099
2100 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) {
2101 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
2102 InstructionMark im(this);
2103 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
2104 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
2105 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2106 emit_int8(0x1D);
2107 emit_operand(src, dst, 1);
2108 emit_int8(imm8);
2109 }
2110
2111 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2112 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
2113 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
2114 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2115 emit_int16(0x13, (0xC0 | encode));
2116 }
2117
2118 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) {
2119 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
2120 InstructionMark im(this);
2121 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
2122 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
2123 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2124 emit_int8(0x13);
2125 emit_operand(dst, src, 0);
2126 }
2127
2128 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
2129 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2130 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2131 emit_int16(0x5B, (0xC0 | encode));
2132 }
2133
2134 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2135 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2136 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2137 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2138 emit_int16(0x5B, (0xC0 | encode));
2139 }
2140
2141 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
2142 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2143 attributes.set_rex_vex_w_reverted();
2144 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2145 emit_int16(0x5A, (0xC0 | encode));
2146 }
2147
2148 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
2149 InstructionMark im(this);
2150 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2151 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2152 attributes.set_rex_vex_w_reverted();
2153 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2154 emit_int8(0x5A);
2155 emit_operand(dst, src, 0);
2156 }
2157
2158 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
2159 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2160 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true);
2161 emit_int16(0x2A, (0xC0 | encode));
2162 }
2163
2164 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
2165 InstructionMark im(this);
2166 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2167 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2168 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2169 emit_int8(0x2A);
2170 emit_operand(dst, src, 0);
2171 }
2172
2173 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
2174 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2175 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes, true);
2176 emit_int16(0x2A, (0xC0 | encode));
2177 }
2178
2179 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
2180 InstructionMark im(this);
2181 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2182 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2183 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2184 emit_int8(0x2A);
2185 emit_operand(dst, src, 0);
2186 }
2187
2188 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
2189 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2190 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes, true);
2191 emit_int16(0x2A, (0xC0 | encode));
2192 }
2193
2194 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
2195 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2196 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2197 emit_int16(0x5A, (0xC0 | encode));
2198 }
2199
2200 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
2201 InstructionMark im(this);
2202 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2203 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2204 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2205 emit_int8(0x5A);
2206 emit_operand(dst, src, 0);
2207 }
2208
2209
2210 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
2211 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2212 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2213 emit_int16(0x2C, (0xC0 | encode));
2214 }
2215
2216 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
2217 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2218 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2219 emit_int16(0x2D, (0xC0 | encode));
2220 }
2221
2222 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
2223 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2224 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2225 emit_int16(0x2C, (0xC0 | encode));
2226 }
2227
2228 void Assembler::evcvttss2sisl(Register dst, XMMRegister src) {
2229 assert(VM_Version::supports_avx10_2(), "");
2230 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2231 attributes.set_is_evex_instruction();
2232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
2233 emit_int16(0x6D, (0xC0 | encode));
2234 }
2235
2236 void Assembler::evcvttss2sisl(Register dst, Address src) {
2237 assert(VM_Version::supports_avx10_2(), "");
2238 InstructionMark im(this);
2239 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2240 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2241 attributes.set_is_evex_instruction();
2242 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
2243 emit_int8((unsigned char)0x6D);
2244 emit_operand(dst, src, 0);
2245 }
2246
2247 void Assembler::evcvttss2sisq(Register dst, XMMRegister src) {
2248 assert(VM_Version::supports_avx10_2(), "");
2249 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2250 attributes.set_is_evex_instruction();
2251 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
2252 emit_int16(0x6D, (0xC0 | encode));
2253 }
2254
2255 void Assembler::evcvttss2sisq(Register dst, Address src) {
2256 assert(VM_Version::supports_avx10_2(), "");
2257 InstructionMark im(this);
2258 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2259 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2260 attributes.set_is_evex_instruction();
2261 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
2262 emit_int8((unsigned char)0x6D);
2263 emit_operand(dst, src, 0);
2264 }
2265
2266 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
2267 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2268 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2269 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2270 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2271 }
2272
2273 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
2274 assert(VM_Version::supports_ssse3(), "");
2275 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2276 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2277 emit_int16(0x1C, (0xC0 | encode));
2278 }
2279
2280 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
2281 assert(VM_Version::supports_ssse3(), "");
2282 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2283 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2284 emit_int16(0x1D, (0xC0 | encode));
2285 }
2286
2287 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
2288 assert(VM_Version::supports_ssse3(), "");
2289 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2290 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2291 emit_int16(0x1E, (0xC0 | encode));
2292 }
2293
2294 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
2295 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2296 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2297 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
2298 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2299 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2300 emit_int16(0x1C, (0xC0 | encode));
2301 }
2302
2303 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
2304 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2305 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2306 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
2307 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2308 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2309 emit_int16(0x1D, (0xC0 | encode));
2310 }
2311
2312 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
2313 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
2314 vector_len == AVX_256bit? VM_Version::supports_avx2() :
2315 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
2316 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2317 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2318 emit_int16(0x1E, (0xC0 | encode));
2319 }
2320
2321 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
2322 assert(UseAVX > 2, "");
2323 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2324 attributes.set_is_evex_instruction();
2325 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2326 emit_int16(0x1F, (0xC0 | encode));
2327 }
2328
2329 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2330 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2331 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2332 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2333 emit_int16(0x5A, (0xC0 | encode));
2334 }
2335
2336 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2337 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2338 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2339 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2340 attributes.set_rex_vex_w_reverted();
2341 emit_int16(0x5A, (0xC0 | encode));
2342 }
2343
2344 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2345 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2346 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2347 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2348 emit_int16(0x5B, (0xC0 | encode));
2349 }
2350
2351 void Assembler::evcvttps2dqs(XMMRegister dst, XMMRegister src, int vector_len) {
2352 assert(VM_Version::supports_avx10_2(), "");
2353 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2354 attributes.set_is_evex_instruction();
2355 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
2356 emit_int16(0x6D, (0xC0 | encode));
2357 }
2358
2359 void Assembler::evcvttps2dqs(XMMRegister dst, Address src, int vector_len) {
2360 assert(VM_Version::supports_avx10_2(), "");
2361 InstructionMark im(this);
2362 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2363 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
2364 attributes.set_is_evex_instruction();
2365 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
2366 emit_int8((unsigned char)0x6D);
2367 emit_operand(dst, src, 0);
2368 }
2369
2370 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2371 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2372 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2373 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2374 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2375 }
2376
2377 void Assembler::evcvttpd2dqs(XMMRegister dst, XMMRegister src, int vector_len) {
2378 assert(VM_Version::supports_avx10_2(), "");
2379 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2380 attributes.set_is_evex_instruction();
2381 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
2382 emit_int16(0x6D, (0xC0 | encode));
2383 }
2384
2385 void Assembler::evcvttpd2dqs(XMMRegister dst, Address src, int vector_len) {
2386 assert(VM_Version::supports_avx10_2(), "");
2387 InstructionMark im(this);
2388 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2389 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
2390 attributes.set_is_evex_instruction();
2391 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
2392 emit_int8((unsigned char)0x6D);
2393 emit_operand(dst, src, 0);
2394 }
2395
2396 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2397 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2398 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2399 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2400 emit_int16(0x5B, (0xC0 | encode));
2401 }
2402
2403 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2404 assert(VM_Version::supports_avx512dq(), "");
2405 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2406 attributes.set_is_evex_instruction();
2407 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2408 emit_int16(0x7A, (0xC0 | encode));
2409 }
2410
2411 void Assembler::evcvttps2qqs(XMMRegister dst, XMMRegister src, int vector_len) {
2412 assert(VM_Version::supports_avx10_2(), "");
2413 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2414 attributes.set_is_evex_instruction();
2415 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
2416 emit_int16(0x6D, (0xC0 | encode));
2417 }
2418
2419 void Assembler::evcvttps2qqs(XMMRegister dst, Address src, int vector_len) {
2420 assert(VM_Version::supports_avx10_2(), "");
2421 InstructionMark im(this);
2422 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2423 attributes.set_address_attributes(/* tuple_type */ EVEX_HV, /* input_size_in_bits */ EVEX_32bit);
2424 attributes.set_is_evex_instruction();
2425 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
2426 emit_int8((unsigned char)0x6D);
2427 emit_operand(dst, src, 0);
2428 }
2429
2430 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2431 assert(VM_Version::supports_avx512dq(), "");
2432 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2433 attributes.set_is_evex_instruction();
2434 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2435 emit_int16(0x7B, (0xC0 | encode));
2436 }
2437
2438 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2439 assert(VM_Version::supports_avx512dq(), "");
2440 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2441 attributes.set_is_evex_instruction();
2442 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2443 emit_int16(0x5B, (0xC0 | encode));
2444 }
2445
2446 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2447 assert(VM_Version::supports_avx512dq(), "");
2448 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2449 attributes.set_is_evex_instruction();
2450 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2451 emit_int16(0x7A, (0xC0 | encode));
2452 }
2453
2454 void Assembler::evcvttpd2qqs(XMMRegister dst, XMMRegister src, int vector_len) {
2455 assert(VM_Version::supports_avx10_2(), "");
2456 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2457 attributes.set_is_evex_instruction();
2458 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
2459 emit_int16(0x6D, (0xC0 | encode));
2460 }
2461
2462 void Assembler::evcvttpd2qqs(XMMRegister dst, Address src, int vector_len) {
2463 assert(VM_Version::supports_avx10_2(), "");
2464 InstructionMark im(this);
2465 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2466 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
2467 attributes.set_is_evex_instruction();
2468 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
2469 emit_int8((unsigned char)0x6D);
2470 emit_operand(dst, src, 0);
2471 }
2472
2473 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2474 assert(VM_Version::supports_avx512dq(), "");
2475 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2476 attributes.set_is_evex_instruction();
2477 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2478 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2479 }
2480
2481 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
2482 assert(VM_Version::supports_avx512bw(), "");
2483 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2484 attributes.set_is_evex_instruction();
2485 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2486 emit_int16(0x30, (0xC0 | encode));
2487 }
2488
2489 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
2490 assert(UseAVX > 2, "");
2491 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2492 attributes.set_is_evex_instruction();
2493 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2494 emit_int16(0x33, (0xC0 | encode));
2495 }
2496
2497 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
2498 assert(UseAVX > 2, "");
2499 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2500 attributes.set_is_evex_instruction();
2501 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2502 emit_int16(0x31, (0xC0 | encode));
2503 }
2504
2505 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
2506 assert(UseAVX > 2, "");
2507 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2508 attributes.set_is_evex_instruction();
2509 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2510 emit_int16(0x35, (0xC0 | encode));
2511 }
2512
2513 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
2514 assert(UseAVX > 2, "");
2515 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2516 attributes.set_is_evex_instruction();
2517 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2518 emit_int16(0x32, (0xC0 | encode));
2519 }
2520
2521 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
2522 assert(UseAVX > 2, "");
2523 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2524 attributes.set_is_evex_instruction();
2525 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2526 emit_int16(0x34, (0xC0 | encode));
2527 }
2528
2529 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
2530 assert(UseAVX > 2, "");
2531 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2532 attributes.set_is_evex_instruction();
2533 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2534 emit_int16(0x25, (0xC0 | encode));
2535 }
2536
2537 void Assembler::decl(Address dst) {
2538 // Don't use it directly. Use MacroAssembler::decrement() instead.
2539 InstructionMark im(this);
2540 prefix(dst);
2541 emit_int8((unsigned char)0xFF);
2542 emit_operand(rcx, dst, 0);
2543 }
2544
2545 void Assembler::edecl(Register dst, Address src, bool no_flags) {
2546 InstructionMark im(this);
2547 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2548 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
2549 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2550 emit_int8((unsigned char)0xFF);
2551 emit_operand(rcx, src, 0);
2552 }
2553
2554 void Assembler::divsd(XMMRegister dst, Address src) {
2555 InstructionMark im(this);
2556 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2557 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2558 attributes.set_rex_vex_w_reverted();
2559 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2560 emit_int8(0x5E);
2561 emit_operand(dst, src, 0);
2562 }
2563
2564 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
2565 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2566 attributes.set_rex_vex_w_reverted();
2567 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2568 emit_int16(0x5E, (0xC0 | encode));
2569 }
2570
2571 void Assembler::divss(XMMRegister dst, Address src) {
2572 InstructionMark im(this);
2573 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2574 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2575 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2576 emit_int8(0x5E);
2577 emit_operand(dst, src, 0);
2578 }
2579
2580 void Assembler::divss(XMMRegister dst, XMMRegister src) {
2581 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2582 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2583 emit_int16(0x5E, (0xC0 | encode));
2584 }
2585
2586 void Assembler::hlt() {
2587 emit_int8((unsigned char)0xF4);
2588 }
2589
2590 void Assembler::idivl(Register src) {
2591 int encode = prefix_and_encode(src->encoding());
2592 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2593 }
2594
2595 void Assembler::eidivl(Register src, bool no_flags) { // Signed
2596 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2597 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2598 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2599 }
2600
2601 void Assembler::divl(Register src) { // Unsigned
2602 int encode = prefix_and_encode(src->encoding());
2603 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2604 }
2605
2606 void Assembler::edivl(Register src, bool no_flags) { // Unsigned
2607 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2608 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2609 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2610 }
2611
2612 void Assembler::imull(Register src) {
2613 int encode = prefix_and_encode(src->encoding());
2614 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2615 }
2616
2617 void Assembler::eimull(Register src, bool no_flags) {
2618 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2619 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2620 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2621 }
2622
2623 void Assembler::imull(Register dst, Register src) {
2624 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
2625 emit_opcode_prefix_and_encoding((unsigned char)0xAF, 0xC0, encode);
2626 }
2627
2628 void Assembler::eimull(Register dst, Register src1, Register src2, bool no_flags) {
2629 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */);
2630 }
2631
2632 void Assembler::imull(Register dst, Address src, int32_t value) {
2633 InstructionMark im(this);
2634 prefix(src, dst);
2635 if (is8bit(value)) {
2636 emit_int8((unsigned char)0x6B);
2637 emit_operand(dst, src, 1);
2638 emit_int8(value);
2639 } else {
2640 emit_int8((unsigned char)0x69);
2641 emit_operand(dst, src, 4);
2642 emit_int32(value);
2643 }
2644 }
2645
2646 void Assembler::eimull(Register dst, Address src, int32_t value, bool no_flags) {
2647 InstructionMark im(this);
2648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2649 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
2650 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2651 if (is8bit(value)) {
2652 emit_int8((unsigned char)0x6B);
2653 emit_operand(dst, src, 1);
2654 emit_int8(value);
2655 } else {
2656 emit_int8((unsigned char)0x69);
2657 emit_operand(dst, src, 4);
2658 emit_int32(value);
2659 }
2660 }
2661
2662 void Assembler::imull(Register dst, Register src, int value) {
2663 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2664 if (is8bit(value)) {
2665 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2666 } else {
2667 emit_int16(0x69, (0xC0 | encode));
2668 emit_int32(value);
2669 }
2670 }
2671
2672 void Assembler::eimull(Register dst, Register src, int value, bool no_flags) {
2673 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2674 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2675 if (is8bit(value)) {
2676 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2677 } else {
2678 emit_int16(0x69, (0xC0 | encode));
2679 emit_int32(value);
2680 }
2681 }
2682
2683 void Assembler::imull(Register dst, Address src) {
2684 InstructionMark im(this);
2685 prefix(src, dst, false, true /* is_map1 */);
2686 emit_int8((unsigned char)0xAF);
2687 emit_operand(dst, src, 0);
2688 }
2689
2690 void Assembler::eimull(Register dst, Register src1, Address src2, bool no_flags) {
2691 InstructionMark im(this);
2692 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, (unsigned char)0xAF, no_flags, true /* is_map1 */);
2693 }
2694
2695 void Assembler::incl(Address dst) {
2696 // Don't use it directly. Use MacroAssembler::increment() instead.
2697 InstructionMark im(this);
2698 prefix(dst);
2699 emit_int8((unsigned char)0xFF);
2700 emit_operand(rax, dst, 0);
2701 }
2702
2703 void Assembler::eincl(Register dst, Address src, bool no_flags) {
2704 // Don't use it directly. Use MacroAssembler::increment() instead.
2705 InstructionMark im(this);
2706 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2707 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
2708 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2709 emit_int8((unsigned char)0xFF);
2710 emit_operand(rax, src, 0);
2711 }
2712
2713 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2714 InstructionMark im(this);
2715 assert((0 <= cc) && (cc < 16), "illegal cc");
2716 if (L.is_bound()) {
2717 address dst = target(L);
2718 assert(dst != nullptr, "jcc most probably wrong");
2719
2720 const int short_size = 2;
2721 const int long_size = 6;
2722 int offs = checked_cast<int>((intptr_t)dst - (intptr_t)pc());
2723 if (maybe_short && is8bit(offs - short_size)) {
2724 // 0111 tttn #8-bit disp
2725 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2726 } else {
2727 // 0000 1111 1000 tttn #32-bit disp
2728 assert(is_simm32(offs - long_size),
2729 "must be 32bit offset (call4)");
2730 emit_int16(0x0F, (0x80 | cc));
2731 emit_int32(offs - long_size);
2732 }
2733 } else {
2734 // Note: could eliminate cond. jumps to this jump if condition
2735 // is the same however, seems to be rather unlikely case.
2736 // Note: use jccb() if label to be bound is very close to get
2737 // an 8-bit displacement
2738 L.add_patch_at(code(), locator());
2739 emit_int16(0x0F, (0x80 | cc));
2740 emit_int32(0);
2741 }
2742 }
2743
2744 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2745 if (L.is_bound()) {
2746 const int short_size = 2;
2747 address entry = target(L);
2748 #ifdef ASSERT
2749 int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size));
2750 int delta = short_branch_delta();
2751 if (delta != 0) {
2752 dist += (dist < 0 ? (-delta) :delta);
2753 }
2754 assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line);
2755 #endif
2756 int offs = checked_cast<int>((intptr_t)entry - (intptr_t)pc());
2757 // 0111 tttn #8-bit disp
2758 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2759 } else {
2760 InstructionMark im(this);
2761 L.add_patch_at(code(), locator(), file, line);
2762 emit_int16(0x70 | cc, 0);
2763 }
2764 }
2765
2766 void Assembler::jmp(Address adr) {
2767 InstructionMark im(this);
2768 prefix(adr);
2769 emit_int8((unsigned char)0xFF);
2770 emit_operand(rsp, adr, 0);
2771 }
2772
2773 void Assembler::jmp(Label& L, bool maybe_short) {
2774 if (L.is_bound()) {
2775 address entry = target(L);
2776 assert(entry != nullptr, "jmp most probably wrong");
2777 InstructionMark im(this);
2778 const int short_size = 2;
2779 const int long_size = 5;
2780 int offs = checked_cast<int>(entry - pc());
2781 if (maybe_short && is8bit(offs - short_size)) {
2782 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
2783 } else {
2784 emit_int8((unsigned char)0xE9);
2785 emit_int32(offs - long_size);
2786 }
2787 } else {
2788 // By default, forward jumps are always 32-bit displacements, since
2789 // we can't yet know where the label will be bound. If you're sure that
2790 // the forward jump will not run beyond 256 bytes, use jmpb to
2791 // force an 8-bit displacement.
2792 InstructionMark im(this);
2793 L.add_patch_at(code(), locator());
2794 emit_int8((unsigned char)0xE9);
2795 emit_int32(0);
2796 }
2797 }
2798
2799 void Assembler::jmp(Register entry) {
2800 int encode = prefix_and_encode(entry->encoding());
2801 emit_int16((unsigned char)0xFF, (0xE0 | encode));
2802 }
2803
2804 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2805 InstructionMark im(this);
2806 emit_int8((unsigned char)0xE9);
2807 assert(dest != nullptr, "must have a target");
2808 intptr_t disp = dest - (pc() + sizeof(int32_t));
2809 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2810 emit_data(checked_cast<int32_t>(disp), rspec, call32_operand);
2811 }
2812
2813 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2814 if (L.is_bound()) {
2815 const int short_size = 2;
2816 address entry = target(L);
2817 assert(entry != nullptr, "jmp most probably wrong");
2818 #ifdef ASSERT
2819 int dist = checked_cast<int>((intptr_t)entry - (intptr_t)(pc() + short_size));
2820 int delta = short_branch_delta();
2821 if (delta != 0) {
2822 dist += (dist < 0 ? (-delta) :delta);
2823 }
2824 assert(is8bit(dist), "Displacement too large for a short jmp at %s:%d", file, line);
2825 #endif
2826 intptr_t offs = entry - pc();
2827 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
2828 } else {
2829 InstructionMark im(this);
2830 L.add_patch_at(code(), locator(), file, line);
2831 emit_int16((unsigned char)0xEB, 0);
2832 }
2833 }
2834
2835 void Assembler::ldmxcsr( Address src) {
2836 // This instruction should be SSE encoded with the REX2 prefix when an
2837 // extended GPR is present. To be consistent when UseAPX is enabled, use
2838 // this encoding even when an extended GPR is not used.
2839 if (UseAVX > 0 && !UseAPX ) {
2840 InstructionMark im(this);
2841 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2842 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2843 emit_int8((unsigned char)0xAE);
2844 emit_operand(as_Register(2), src, 0);
2845 } else {
2846 InstructionMark im(this);
2847 prefix(src, true /* is_map1 */);
2848 emit_int8((unsigned char)0xAE);
2849 emit_operand(as_Register(2), src, 0);
2850 }
2851 }
2852
2853 void Assembler::leal(Register dst, Address src) {
2854 InstructionMark im(this);
2855 prefix(src, dst);
2856 emit_int8((unsigned char)0x8D);
2857 emit_operand(dst, src, 0);
2858 }
2859
2860 void Assembler::lea(Register dst, Label& L) {
2861 emit_prefix_and_int8(get_prefixq(Address(), dst), (unsigned char)0x8D);
2862 if (!L.is_bound()) {
2863 // Patch @0x8D opcode
2864 L.add_patch_at(code(), CodeBuffer::locator(offset() - 1, sect()));
2865 // Register and [rip+disp] operand
2866 emit_modrm(0b00, raw_encode(dst), 0b101);
2867 emit_int32(0);
2868 } else {
2869 // Register and [rip+disp] operand
2870 emit_modrm(0b00, raw_encode(dst), 0b101);
2871 // Adjust displacement by sizeof lea instruction
2872 int32_t disp = checked_cast<int32_t>(target(L) - (pc() + sizeof(int32_t)));
2873 assert(is_simm32(disp), "must be 32bit offset [rip+offset]");
2874 emit_int32(disp);
2875 }
2876 }
2877
2878 void Assembler::lfence() {
2879 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
2880 }
2881
2882 void Assembler::lock() {
2883 emit_int8((unsigned char)0xF0);
2884 }
2885
2886 void Assembler::size_prefix() {
2887 emit_int8(0x66);
2888 }
2889
2890 void Assembler::lzcntl(Register dst, Register src) {
2891 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2892 emit_int8((unsigned char)0xF3);
2893 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
2894 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode);
2895 }
2896
2897 void Assembler::elzcntl(Register dst, Register src, bool no_flags) {
2898 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2899 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2900 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2901 emit_int16((unsigned char)0xF5, (0xC0 | encode));
2902 }
2903
2904 void Assembler::lzcntl(Register dst, Address src) {
2905 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2906 InstructionMark im(this);
2907 emit_int8((unsigned char)0xF3);
2908 prefix(src, dst, false, true /* is_map1 */);
2909 emit_int8((unsigned char)0xBD);
2910 emit_operand(dst, src, 0);
2911 }
2912
2913 void Assembler::elzcntl(Register dst, Address src, bool no_flags) {
2914 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2915 InstructionMark im(this);
2916 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2917 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
2918 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
2919 emit_int8((unsigned char)0xF5);
2920 emit_operand(dst, src, 0);
2921 }
2922
2923 // Emit mfence instruction
2924 void Assembler::mfence() {
2925 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
2926 }
2927
2928 // Emit sfence instruction
2929 void Assembler::sfence() {
2930 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
2931 }
2932
2933 void Assembler::mov(Register dst, Register src) {
2934 movq(dst, src);
2935 }
2936
2937 void Assembler::movapd(XMMRegister dst, Address src) {
2938 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2939 InstructionMark im(this);
2940 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2941 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2942 attributes.set_rex_vex_w_reverted();
2943 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2944 emit_int8(0x28);
2945 emit_operand(dst, src, 0);
2946 }
2947
2948 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2949 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2950 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2951 attributes.set_rex_vex_w_reverted();
2952 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2953 emit_int16(0x28, (0xC0 | encode));
2954 }
2955
2956 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2957 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2958 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2959 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2960 emit_int16(0x28, (0xC0 | encode));
2961 }
2962
2963 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2964 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2965 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2966 emit_int16(0x16, (0xC0 | encode));
2967 }
2968
2969 void Assembler::movb(Register dst, Address src) {
2970 InstructionMark im(this);
2971 prefix(src, dst, true);
2972 emit_int8((unsigned char)0x8A);
2973 emit_operand(dst, src, 0);
2974 }
2975
2976 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2977 assert(VM_Version::supports_sse3(), "");
2978 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2979 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2980 attributes.set_rex_vex_w_reverted();
2981 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2982 emit_int16(0x12, 0xC0 | encode);
2983 }
2984
2985 void Assembler::movddup(XMMRegister dst, Address src) {
2986 assert(VM_Version::supports_sse3(), "");
2987 InstructionMark im(this);
2988 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2989 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
2990 attributes.set_rex_vex_w_reverted();
2991 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2992 emit_int8(0x12);
2993 emit_operand(dst, src, 0);
2994 }
2995
2996 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
2997 assert(VM_Version::supports_avx(), "");
2998 InstructionMark im(this);
2999 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3000 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
3001 attributes.set_rex_vex_w_reverted();
3002 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3003 emit_int8(0x12);
3004 emit_operand(dst, src, 0);
3005 }
3006
3007 void Assembler::kmovbl(KRegister dst, KRegister src) {
3008 assert(VM_Version::supports_avx512dq(), "");
3009 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3010 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3011 emit_int16((unsigned char)0x90, (0xC0 | encode));
3012 }
3013
3014 void Assembler::kmovbl(KRegister dst, Register src) {
3015 assert(VM_Version::supports_avx512dq(), "");
3016 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3017 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
3018 emit_int16((unsigned char)0x92, (0xC0 | encode));
3019 }
3020
3021 void Assembler::kmovbl(Register dst, KRegister src) {
3022 assert(VM_Version::supports_avx512dq(), "");
3023 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3024 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3025 emit_int16((unsigned char)0x93, (0xC0 | encode));
3026 }
3027
3028 void Assembler::kmovwl(KRegister dst, Register src) {
3029 assert(VM_Version::supports_evex(), "");
3030 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3031 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes, true);
3032 emit_int16((unsigned char)0x92, (0xC0 | encode));
3033 }
3034
3035 void Assembler::kmovwl(Register dst, KRegister src) {
3036 assert(VM_Version::supports_evex(), "");
3037 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3038 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3039 emit_int16((unsigned char)0x93, (0xC0 | encode));
3040 }
3041
3042 void Assembler::kmovwl(KRegister dst, Address src) {
3043 assert(VM_Version::supports_evex(), "");
3044 InstructionMark im(this);
3045 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3046 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
3047 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3048 emit_int8((unsigned char)0x90);
3049 emit_operand(dst, src, 0);
3050 }
3051
3052 void Assembler::kmovwl(Address dst, KRegister src) {
3053 assert(VM_Version::supports_evex(), "");
3054 InstructionMark im(this);
3055 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3056 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
3057 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3058 emit_int8((unsigned char)0x91);
3059 emit_operand(src, dst, 0);
3060 }
3061
3062 void Assembler::kmovwl(KRegister dst, KRegister src) {
3063 assert(VM_Version::supports_evex(), "");
3064 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3065 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3066 emit_int16((unsigned char)0x90, (0xC0 | encode));
3067 }
3068
3069 void Assembler::kmovdl(KRegister dst, Register src) {
3070 assert(VM_Version::supports_avx512bw(), "");
3071 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3072 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true);
3073 emit_int16((unsigned char)0x92, (0xC0 | encode));
3074 }
3075
3076 void Assembler::kmovdl(Register dst, KRegister src) {
3077 assert(VM_Version::supports_avx512bw(), "");
3078 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3079 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3080 emit_int16((unsigned char)0x93, (0xC0 | encode));
3081 }
3082
3083 void Assembler::kmovql(KRegister dst, KRegister src) {
3084 assert(VM_Version::supports_avx512bw(), "");
3085 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3086 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3087 emit_int16((unsigned char)0x90, (0xC0 | encode));
3088 }
3089
3090 void Assembler::kmovql(KRegister dst, Address src) {
3091 assert(VM_Version::supports_avx512bw(), "");
3092 InstructionMark im(this);
3093 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3094 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
3095 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3096 emit_int8((unsigned char)0x90);
3097 emit_operand(dst, src, 0);
3098 }
3099
3100 void Assembler::kmovql(Address dst, KRegister src) {
3101 assert(VM_Version::supports_avx512bw(), "");
3102 InstructionMark im(this);
3103 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3104 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
3105 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3106 emit_int8((unsigned char)0x91);
3107 emit_operand(src, dst, 0);
3108 }
3109
3110 void Assembler::kmovql(KRegister dst, Register src) {
3111 assert(VM_Version::supports_avx512bw(), "");
3112 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3113 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true);
3114 emit_int16((unsigned char)0x92, (0xC0 | encode));
3115 }
3116
3117 void Assembler::kmovql(Register dst, KRegister src) {
3118 assert(VM_Version::supports_avx512bw(), "");
3119 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3120 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3121 emit_int16((unsigned char)0x93, (0xC0 | encode));
3122 }
3123
3124 void Assembler::knotwl(KRegister dst, KRegister src) {
3125 assert(VM_Version::supports_evex(), "");
3126 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3127 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3128 emit_int16(0x44, (0xC0 | encode));
3129 }
3130
3131 void Assembler::knotbl(KRegister dst, KRegister src) {
3132 assert(VM_Version::supports_evex(), "");
3133 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3134 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3135 emit_int16(0x44, (0xC0 | encode));
3136 }
3137
3138 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
3139 assert(VM_Version::supports_avx512dq(), "");
3140 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3141 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3142 emit_int16(0x45, (0xC0 | encode));
3143 }
3144
3145 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
3146 assert(VM_Version::supports_evex(), "");
3147 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3148 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3149 emit_int16(0x45, (0xC0 | encode));
3150 }
3151
3152 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
3153 assert(VM_Version::supports_avx512bw(), "");
3154 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3155 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3156 emit_int16(0x45, (0xC0 | encode));
3157 }
3158
3159 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
3160 assert(VM_Version::supports_avx512bw(), "");
3161 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3162 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3163 emit_int16(0x45, (0xC0 | encode));
3164 }
3165
3166 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
3167 assert(VM_Version::supports_avx512dq(), "");
3168 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3169 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3170 emit_int16(0x47, (0xC0 | encode));
3171 }
3172
3173 void Assembler::kxnorwl(KRegister dst, KRegister src1, KRegister src2) {
3174 assert(VM_Version::supports_evex(), "");
3175 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3176 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3177 emit_int16(0x46, (0xC0 | encode));
3178 }
3179
3180 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
3181 assert(VM_Version::supports_evex(), "");
3182 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3183 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3184 emit_int16(0x47, (0xC0 | encode));
3185 }
3186
3187 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
3188 assert(VM_Version::supports_avx512bw(), "");
3189 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3190 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3191 emit_int16(0x47, (0xC0 | encode));
3192 }
3193
3194 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
3195 assert(VM_Version::supports_avx512bw(), "");
3196 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3197 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3198 emit_int16(0x47, (0xC0 | encode));
3199 }
3200
3201 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
3202 assert(VM_Version::supports_avx512dq(), "");
3203 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3204 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3205 emit_int16(0x41, (0xC0 | encode));
3206 }
3207
3208 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
3209 assert(VM_Version::supports_evex(), "");
3210 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3211 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3212 emit_int16(0x41, (0xC0 | encode));
3213 }
3214
3215 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
3216 assert(VM_Version::supports_avx512bw(), "");
3217 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3218 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3219 emit_int16(0x41, (0xC0 | encode));
3220 }
3221
3222 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
3223 assert(VM_Version::supports_avx512bw(), "");
3224 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3225 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3226 emit_int16(0x41, (0xC0 | encode));
3227 }
3228
3229 void Assembler::knotdl(KRegister dst, KRegister src) {
3230 assert(VM_Version::supports_avx512bw(), "");
3231 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3233 emit_int16(0x44, (0xC0 | encode));
3234 }
3235
3236 void Assembler::knotql(KRegister dst, KRegister src) {
3237 assert(VM_Version::supports_avx512bw(), "");
3238 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3239 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3240 emit_int16(0x44, (0xC0 | encode));
3241 }
3242
3243 // This instruction produces ZF or CF flags
3244 void Assembler::kortestbl(KRegister src1, KRegister src2) {
3245 assert(VM_Version::supports_avx512dq(), "");
3246 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3247 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3248 emit_int16((unsigned char)0x98, (0xC0 | encode));
3249 }
3250
3251 // This instruction produces ZF or CF flags
3252 void Assembler::kortestwl(KRegister src1, KRegister src2) {
3253 assert(VM_Version::supports_evex(), "");
3254 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3255 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3256 emit_int16((unsigned char)0x98, (0xC0 | encode));
3257 }
3258
3259 // This instruction produces ZF or CF flags
3260 void Assembler::kortestdl(KRegister src1, KRegister src2) {
3261 assert(VM_Version::supports_avx512bw(), "");
3262 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3263 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3264 emit_int16((unsigned char)0x98, (0xC0 | encode));
3265 }
3266
3267 // This instruction produces ZF or CF flags
3268 void Assembler::kortestql(KRegister src1, KRegister src2) {
3269 assert(VM_Version::supports_avx512bw(), "");
3270 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3271 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3272 emit_int16((unsigned char)0x98, (0xC0 | encode));
3273 }
3274
3275 // This instruction produces ZF or CF flags
3276 void Assembler::ktestql(KRegister src1, KRegister src2) {
3277 assert(VM_Version::supports_avx512bw(), "");
3278 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3279 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3280 emit_int16((unsigned char)0x99, (0xC0 | encode));
3281 }
3282
3283 void Assembler::ktestdl(KRegister src1, KRegister src2) {
3284 assert(VM_Version::supports_avx512bw(), "");
3285 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3286 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3287 emit_int16((unsigned char)0x99, (0xC0 | encode));
3288 }
3289
3290 void Assembler::ktestwl(KRegister src1, KRegister src2) {
3291 assert(VM_Version::supports_avx512dq(), "");
3292 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3293 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3294 emit_int16((unsigned char)0x99, (0xC0 | encode));
3295 }
3296
3297 void Assembler::ktestbl(KRegister src1, KRegister src2) {
3298 assert(VM_Version::supports_avx512dq(), "");
3299 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3300 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3301 emit_int16((unsigned char)0x99, (0xC0 | encode));
3302 }
3303
3304 void Assembler::ktestq(KRegister src1, KRegister src2) {
3305 assert(VM_Version::supports_avx512bw(), "");
3306 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3307 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3308 emit_int16((unsigned char)0x99, (0xC0 | encode));
3309 }
3310
3311 void Assembler::ktestd(KRegister src1, KRegister src2) {
3312 assert(VM_Version::supports_avx512bw(), "");
3313 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3314 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3315 emit_int16((unsigned char)0x99, (0xC0 | encode));
3316 }
3317
3318 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
3319 assert(VM_Version::supports_avx512dq(), "");
3320 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3321 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3322 emit_int16(0x46, (0xC0 | encode));
3323 }
3324
3325 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
3326 assert(VM_Version::supports_avx512dq(), "");
3327 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3328 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3329 emit_int16(0x32, (0xC0 | encode));
3330 emit_int8(imm8);
3331 }
3332
3333 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
3334 assert(VM_Version::supports_avx512bw(), "");
3335 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3336 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3337 emit_int16(0x33, (0xC0 | encode));
3338 emit_int8(imm8);
3339 }
3340
3341
3342 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
3343 assert(VM_Version::supports_avx512dq(), "");
3344 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3345 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3346 emit_int16(0x30, (0xC0 | encode));
3347 }
3348
3349 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
3350 assert(VM_Version::supports_evex(), "");
3351 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3352 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3353 emit_int16(0x30, (0xC0 | encode));
3354 emit_int8(imm8);
3355 }
3356
3357 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
3358 assert(VM_Version::supports_avx512bw(), "");
3359 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3360 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3361 emit_int16(0x31, (0xC0 | encode));
3362 emit_int8(imm8);
3363 }
3364
3365 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
3366 assert(VM_Version::supports_avx512bw(), "");
3367 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3368 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3369 emit_int16(0x31, (0xC0 | encode));
3370 emit_int8(imm8);
3371 }
3372
3373 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
3374 assert(VM_Version::supports_avx512bw(), "");
3375 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3376 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3377 emit_int16(0x4B, (0xC0 | encode));
3378 }
3379
3380 void Assembler::movb(Address dst, int imm8) {
3381 InstructionMark im(this);
3382 prefix(dst);
3383 emit_int8((unsigned char)0xC6);
3384 emit_operand(rax, dst, 1);
3385 emit_int8(imm8);
3386 }
3387
3388
3389 void Assembler::movb(Address dst, Register src) {
3390 assert(src->has_byte_register(), "must have byte register");
3391 InstructionMark im(this);
3392 prefix(dst, src, true);
3393 emit_int8((unsigned char)0x88);
3394 emit_operand(src, dst, 0);
3395 }
3396
3397 void Assembler::movdl(XMMRegister dst, Register src) {
3398 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3399 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
3400 emit_int16(0x6E, (0xC0 | encode));
3401 }
3402
3403 void Assembler::movdl(Register dst, XMMRegister src) {
3404 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3405 // swap src/dst to get correct prefix
3406 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
3407 emit_int16(0x7E, (0xC0 | encode));
3408 }
3409
3410 void Assembler::movdl(XMMRegister dst, Address src) {
3411 InstructionMark im(this);
3412 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3413 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3414 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3415 emit_int8(0x6E);
3416 emit_operand(dst, src, 0);
3417 }
3418
3419 void Assembler::movdl(Address dst, XMMRegister src) {
3420 InstructionMark im(this);
3421 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3422 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3423 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3424 emit_int8(0x7E);
3425 emit_operand(src, dst, 0);
3426 }
3427
3428 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
3429 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3430 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3431 emit_int16(0x6F, (0xC0 | encode));
3432 }
3433
3434 void Assembler::movdqa(XMMRegister dst, Address src) {
3435 InstructionMark im(this);
3436 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3437 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3438 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3439 emit_int8(0x6F);
3440 emit_operand(dst, src, 0);
3441 }
3442
3443 void Assembler::movdqu(XMMRegister dst, Address src) {
3444 InstructionMark im(this);
3445 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3446 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3447 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3448 emit_int8(0x6F);
3449 emit_operand(dst, src, 0);
3450 }
3451
3452 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
3453 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3454 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3455 emit_int16(0x6F, (0xC0 | encode));
3456 }
3457
3458 void Assembler::movdqu(Address dst, XMMRegister src) {
3459 InstructionMark im(this);
3460 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3461 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3462 attributes.reset_is_clear_context();
3463 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3464 emit_int8(0x7F);
3465 emit_operand(src, dst, 0);
3466 }
3467
3468 // Move Unaligned 256bit Vector
3469 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3470 assert(UseAVX > 0, "");
3471 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3472 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3473 emit_int16(0x6F, (0xC0 | encode));
3474 }
3475
3476 void Assembler::vmovw(XMMRegister dst, Register src) {
3477 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
3478 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3479 attributes.set_is_evex_instruction();
3480 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes, true);
3481 emit_int16(0x6E, (0xC0 | encode));
3482 }
3483
3484 void Assembler::vmovw(Register dst, XMMRegister src) {
3485 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
3486 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3487 attributes.set_is_evex_instruction();
3488 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes, true);
3489 emit_int16(0x7E, (0xC0 | encode));
3490 }
3491
3492 void Assembler::vmovdqu(XMMRegister dst, Address src) {
3493 assert(UseAVX > 0, "");
3494 InstructionMark im(this);
3495 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3496 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3497 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3498 emit_int8(0x6F);
3499 emit_operand(dst, src, 0);
3500 }
3501
3502 void Assembler::vmovdqu(Address dst, XMMRegister src) {
3503 assert(UseAVX > 0, "");
3504 InstructionMark im(this);
3505 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3506 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3507 attributes.reset_is_clear_context();
3508 // swap src<->dst for encoding
3509 assert(src != xnoreg, "sanity");
3510 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3511 emit_int8(0x7F);
3512 emit_operand(src, dst, 0);
3513 }
3514
3515 // Move Aligned 256bit Vector
3516 void Assembler::vmovdqa(XMMRegister dst, Address src) {
3517 assert(UseAVX > 0, "");
3518 InstructionMark im(this);
3519 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3520 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3521 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3522 emit_int8(0x6F);
3523 emit_operand(dst, src, 0);
3524 }
3525
3526 void Assembler::vmovdqa(Address dst, XMMRegister src) {
3527 assert(UseAVX > 0, "");
3528 InstructionMark im(this);
3529 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3530 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3531 attributes.reset_is_clear_context();
3532 // swap src<->dst for encoding
3533 assert(src != xnoreg, "sanity");
3534 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3535 emit_int8(0x7F);
3536 emit_operand(src, dst, 0);
3537 }
3538
3539 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3540 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3541 InstructionMark im(this);
3542 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3543 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3544 emit_int8((unsigned char)0x8C);
3545 emit_operand(dst, src, 0);
3546 }
3547
3548 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3549 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3550 InstructionMark im(this);
3551 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3552 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3553 emit_int8((unsigned char)0x8C);
3554 emit_operand(dst, src, 0);
3555 }
3556
3557 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3558 assert(UseAVX > 0, "requires some form of AVX");
3559 InstructionMark im(this);
3560 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3561 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3562 emit_int8(0x2C);
3563 emit_operand(dst, src, 0);
3564 }
3565
3566 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3567 assert(UseAVX > 0, "requires some form of AVX");
3568 InstructionMark im(this);
3569 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3570 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3571 emit_int8(0x2D);
3572 emit_operand(dst, src, 0);
3573 }
3574
3575 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3576 assert(UseAVX > 0, "");
3577 InstructionMark im(this);
3578 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3579 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3580 emit_int8(0x2E);
3581 emit_operand(src, dst, 0);
3582 }
3583
3584 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3585 assert(UseAVX > 0, "");
3586 InstructionMark im(this);
3587 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3588 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3589 emit_int8(0x2F);
3590 emit_operand(src, dst, 0);
3591 }
3592
3593 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
3594 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3595 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3596 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3597 attributes.set_embedded_opmask_register_specifier(mask);
3598 attributes.set_is_evex_instruction();
3599 if (merge) {
3600 attributes.reset_is_clear_context();
3601 }
3602 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3603 emit_int16(0x6F, (0xC0 | encode));
3604 }
3605
3606 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
3607 // Unmasked instruction
3608 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3609 }
3610
3611 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3612 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3613 InstructionMark im(this);
3614 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3615 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3616 attributes.set_embedded_opmask_register_specifier(mask);
3617 attributes.set_is_evex_instruction();
3618 if (merge) {
3619 attributes.reset_is_clear_context();
3620 }
3621 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3622 emit_int8(0x6F);
3623 emit_operand(dst, src, 0);
3624 }
3625
3626 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
3627 // Unmasked instruction
3628 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3629 }
3630
3631 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3632 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3633 assert(src != xnoreg, "sanity");
3634 InstructionMark im(this);
3635 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3636 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3637 attributes.set_embedded_opmask_register_specifier(mask);
3638 attributes.set_is_evex_instruction();
3639 if (merge) {
3640 attributes.reset_is_clear_context();
3641 }
3642 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3643 emit_int8(0x7F);
3644 emit_operand(src, dst, 0);
3645 }
3646
3647 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
3648 // Unmasked instruction
3649 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3650 }
3651
3652 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
3653 // Unmasked instruction
3654 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3655 }
3656
3657 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3658 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3659 InstructionMark im(this);
3660 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3661 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3662 attributes.set_embedded_opmask_register_specifier(mask);
3663 attributes.set_is_evex_instruction();
3664 if (merge) {
3665 attributes.reset_is_clear_context();
3666 }
3667 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3668 emit_int8(0x6F);
3669 emit_operand(dst, src, 0);
3670 }
3671
3672 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
3673 // Unmasked instruction
3674 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3675 }
3676
3677 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3678 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3679 assert(src != xnoreg, "sanity");
3680 InstructionMark im(this);
3681 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3682 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3683 attributes.set_embedded_opmask_register_specifier(mask);
3684 attributes.set_is_evex_instruction();
3685 if (merge) {
3686 attributes.reset_is_clear_context();
3687 }
3688 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3689 emit_int8(0x7F);
3690 emit_operand(src, dst, 0);
3691 }
3692
3693 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3694 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
3695 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3696 attributes.set_embedded_opmask_register_specifier(mask);
3697 attributes.set_is_evex_instruction();
3698 if (merge) {
3699 attributes.reset_is_clear_context();
3700 }
3701 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3702 emit_int16(0x6F, (0xC0 | encode));
3703 }
3704
3705
3706 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
3707 // Unmasked instruction
3708 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3709 }
3710
3711 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3712 assert(VM_Version::supports_evex(), "");
3713 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3714 attributes.set_embedded_opmask_register_specifier(mask);
3715 attributes.set_is_evex_instruction();
3716 if (merge) {
3717 attributes.reset_is_clear_context();
3718 }
3719 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3720 emit_int16(0x6F, (0xC0 | encode));
3721 }
3722
3723 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
3724 // Unmasked instruction
3725 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3726 }
3727
3728 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3729 assert(VM_Version::supports_evex(), "");
3730 InstructionMark im(this);
3731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
3732 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3733 attributes.set_embedded_opmask_register_specifier(mask);
3734 attributes.set_is_evex_instruction();
3735 if (merge) {
3736 attributes.reset_is_clear_context();
3737 }
3738 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3739 emit_int8(0x6F);
3740 emit_operand(dst, src, 0);
3741 }
3742
3743 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
3744 // Unmasked isntruction
3745 evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
3746 }
3747
3748 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3749 assert(VM_Version::supports_evex(), "");
3750 assert(src != xnoreg, "sanity");
3751 InstructionMark im(this);
3752 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3753 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3754 attributes.set_embedded_opmask_register_specifier(mask);
3755 attributes.set_is_evex_instruction();
3756 if (merge) {
3757 attributes.reset_is_clear_context();
3758 }
3759 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3760 emit_int8(0x7F);
3761 emit_operand(src, dst, 0);
3762 }
3763
3764 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
3765 // Unmasked instruction
3766 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3767 }
3768
3769 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3770 assert(VM_Version::supports_evex(), "");
3771 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3772 attributes.set_embedded_opmask_register_specifier(mask);
3773 attributes.set_is_evex_instruction();
3774 if (merge) {
3775 attributes.reset_is_clear_context();
3776 }
3777 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3778 emit_int16(0x6F, (0xC0 | encode));
3779 }
3780
3781 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
3782 // Unmasked instruction
3783 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3784 }
3785
3786 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3787 assert(VM_Version::supports_evex(), "");
3788 InstructionMark im(this);
3789 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3790 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3791 attributes.set_embedded_opmask_register_specifier(mask);
3792 attributes.set_is_evex_instruction();
3793 if (merge) {
3794 attributes.reset_is_clear_context();
3795 }
3796 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3797 emit_int8(0x6F);
3798 emit_operand(dst, src, 0);
3799 }
3800
3801 // Move Aligned 512bit Vector
3802 void Assembler::evmovdqaq(XMMRegister dst, Address src, int vector_len) {
3803 // Unmasked instruction
3804 evmovdqaq(dst, k0, src, /*merge*/ false, vector_len);
3805 }
3806
3807 void Assembler::evmovdqaq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3808 assert(VM_Version::supports_evex(), "");
3809 InstructionMark im(this);
3810 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3811 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3812 attributes.set_embedded_opmask_register_specifier(mask);
3813 attributes.set_is_evex_instruction();
3814 if (merge) {
3815 attributes.reset_is_clear_context();
3816 }
3817 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3818 emit_int8(0x6F);
3819 emit_operand(dst, src, 0);
3820 }
3821
3822 void Assembler::evmovntdquq(Address dst, XMMRegister src, int vector_len) {
3823 // Unmasked instruction
3824 evmovntdquq(dst, k0, src, /*merge*/ true, vector_len);
3825 }
3826
3827 void Assembler::evmovntdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3828 assert(VM_Version::supports_evex(), "");
3829 assert(src != xnoreg, "sanity");
3830 InstructionMark im(this);
3831 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3832 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3833 attributes.set_embedded_opmask_register_specifier(mask);
3834 if (merge) {
3835 attributes.reset_is_clear_context();
3836 }
3837 attributes.set_is_evex_instruction();
3838 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3839 emit_int8(0xE7);
3840 emit_operand(src, dst, 0);
3841 }
3842
3843 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
3844 // Unmasked instruction
3845 evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
3846 }
3847
3848 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3849 assert(VM_Version::supports_evex(), "");
3850 assert(src != xnoreg, "sanity");
3851 InstructionMark im(this);
3852 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3853 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3854 attributes.set_embedded_opmask_register_specifier(mask);
3855 if (merge) {
3856 attributes.reset_is_clear_context();
3857 }
3858 attributes.set_is_evex_instruction();
3859 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3860 emit_int8(0x7F);
3861 emit_operand(src, dst, 0);
3862 }
3863
3864 // Uses zero extension on 64bit
3865
3866 void Assembler::movl(Register dst, int32_t imm32) {
3867 int encode = prefix_and_encode(dst->encoding());
3868 emit_int8(0xB8 | encode);
3869 emit_int32(imm32);
3870 }
3871
3872 void Assembler::movl(Register dst, Register src) {
3873 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3874 emit_int16((unsigned char)0x8B, (0xC0 | encode));
3875 }
3876
3877 void Assembler::movl(Register dst, Address src) {
3878 InstructionMark im(this);
3879 prefix(src, dst);
3880 emit_int8((unsigned char)0x8B);
3881 emit_operand(dst, src, 0);
3882 }
3883
3884 void Assembler::movl(Address dst, int32_t imm32) {
3885 InstructionMark im(this);
3886 prefix(dst);
3887 emit_int8((unsigned char)0xC7);
3888 emit_operand(rax, dst, 4);
3889 emit_int32(imm32);
3890 }
3891
3892 void Assembler::movl(Address dst, Register src) {
3893 InstructionMark im(this);
3894 prefix(dst, src);
3895 emit_int8((unsigned char)0x89);
3896 emit_operand(src, dst, 0);
3897 }
3898
3899 // New cpus require to use movsd and movss to avoid partial register stall
3900 // when loading from memory. But for old Opteron use movlpd instead of movsd.
3901 // The selection is done in MacroAssembler::movdbl() and movflt().
3902 void Assembler::movlpd(XMMRegister dst, Address src) {
3903 InstructionMark im(this);
3904 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3905 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3906 attributes.set_rex_vex_w_reverted();
3907 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3908 emit_int8(0x12);
3909 emit_operand(dst, src, 0);
3910 }
3911
3912 void Assembler::movq(XMMRegister dst, Address src) {
3913 InstructionMark im(this);
3914 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3915 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3916 attributes.set_rex_vex_w_reverted();
3917 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3918 emit_int8(0x7E);
3919 emit_operand(dst, src, 0);
3920 }
3921
3922 void Assembler::movq(Address dst, XMMRegister src) {
3923 InstructionMark im(this);
3924 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3925 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3926 attributes.set_rex_vex_w_reverted();
3927 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3928 emit_int8((unsigned char)0xD6);
3929 emit_operand(src, dst, 0);
3930 }
3931
3932 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3933 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3934 attributes.set_rex_vex_w_reverted();
3935 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3936 emit_int16((unsigned char)0xD6, (0xC0 | encode));
3937 }
3938
3939 void Assembler::movq(Register dst, XMMRegister src) {
3940 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3941 // swap src/dst to get correct prefix
3942 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
3943 emit_int16(0x7E, (0xC0 | encode));
3944 }
3945
3946 void Assembler::movq(XMMRegister dst, Register src) {
3947 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3948 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
3949 emit_int16(0x6E, (0xC0 | encode));
3950 }
3951
3952 void Assembler::movsbl(Register dst, Address src) { // movsxb
3953 InstructionMark im(this);
3954 prefix(src, dst, false, true /* is_map1 */);
3955 emit_int8((unsigned char)0xBE);
3956 emit_operand(dst, src, 0);
3957 }
3958
3959 void Assembler::movsbl(Register dst, Register src) { // movsxb
3960 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true, true /* is_map1 */);
3961 emit_opcode_prefix_and_encoding((unsigned char)0xBE, 0xC0, encode);
3962 }
3963
3964 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3965 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3966 attributes.set_rex_vex_w_reverted();
3967 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3968 emit_int16(0x10, (0xC0 | encode));
3969 }
3970
3971 void Assembler::movsd(XMMRegister dst, Address src) {
3972 InstructionMark im(this);
3973 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3974 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3975 attributes.set_rex_vex_w_reverted();
3976 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3977 emit_int8(0x10);
3978 emit_operand(dst, src, 0);
3979 }
3980
3981 void Assembler::movsd(Address dst, XMMRegister src) {
3982 InstructionMark im(this);
3983 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3984 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3985 attributes.reset_is_clear_context();
3986 attributes.set_rex_vex_w_reverted();
3987 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3988 emit_int8(0x11);
3989 emit_operand(src, dst, 0);
3990 }
3991
3992 void Assembler::vmovsd(XMMRegister dst, XMMRegister src, XMMRegister src2) {
3993 assert(UseAVX > 0, "Requires some form of AVX");
3994 InstructionMark im(this);
3995 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3996 int encode = vex_prefix_and_encode(src2->encoding(), src->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3997 emit_int16(0x11, (0xC0 | encode));
3998 }
3999
4000 void Assembler::movss(XMMRegister dst, XMMRegister src) {
4001 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4002 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4003 emit_int16(0x10, (0xC0 | encode));
4004 }
4005
4006 void Assembler::movss(XMMRegister dst, Address src) {
4007 InstructionMark im(this);
4008 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4010 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4011 emit_int8(0x10);
4012 emit_operand(dst, src, 0);
4013 }
4014
4015 void Assembler::movss(Address dst, XMMRegister src) {
4016 InstructionMark im(this);
4017 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4018 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4019 attributes.reset_is_clear_context();
4020 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4021 emit_int8(0x11);
4022 emit_operand(src, dst, 0);
4023 }
4024
4025 void Assembler::movswl(Register dst, Address src) { // movsxw
4026 InstructionMark im(this);
4027 prefix(src, dst, false, true /* is_map1 */);
4028 emit_int8((unsigned char)0xBF);
4029 emit_operand(dst, src, 0);
4030 }
4031
4032 void Assembler::movswl(Register dst, Register src) { // movsxw
4033 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
4034 emit_opcode_prefix_and_encoding((unsigned char)0xBF, 0xC0, encode);
4035 }
4036
4037 void Assembler::movups(XMMRegister dst, Address src) {
4038 InstructionMark im(this);
4039 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4040 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4041 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4042 emit_int8(0x10);
4043 emit_operand(dst, src, 0);
4044 }
4045
4046 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
4047 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
4048 InstructionMark im(this);
4049 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4050 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4051 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4052 emit_int8(0x10);
4053 emit_operand(dst, src, 0);
4054 }
4055
4056 void Assembler::movups(Address dst, XMMRegister src) {
4057 InstructionMark im(this);
4058 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4059 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4060 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4061 emit_int8(0x11);
4062 emit_operand(src, dst, 0);
4063 }
4064
4065 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
4066 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
4067 InstructionMark im(this);
4068 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4069 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4070 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4071 emit_int8(0x11);
4072 emit_operand(src, dst, 0);
4073 }
4074
4075 void Assembler::movw(Address dst, int imm16) {
4076 InstructionMark im(this);
4077
4078 emit_int8(0x66); // switch to 16-bit mode
4079 prefix(dst);
4080 emit_int8((unsigned char)0xC7);
4081 emit_operand(rax, dst, 2);
4082 emit_int16(imm16);
4083 }
4084
4085 void Assembler::movw(Register dst, Address src) {
4086 InstructionMark im(this);
4087 emit_int8(0x66);
4088 prefix(src, dst);
4089 emit_int8((unsigned char)0x8B);
4090 emit_operand(dst, src, 0);
4091 }
4092
4093 void Assembler::movw(Address dst, Register src) {
4094 InstructionMark im(this);
4095 emit_int8(0x66);
4096 prefix(dst, src);
4097 emit_int8((unsigned char)0x89);
4098 emit_operand(src, dst, 0);
4099 }
4100
4101 void Assembler::movzbl(Register dst, Address src) { // movzxb
4102 InstructionMark im(this);
4103 prefix(src, dst, false, true /* is_map1 */);
4104 emit_int8((unsigned char)0xB6);
4105 emit_operand(dst, src, 0);
4106 }
4107
4108 void Assembler::movzbl(Register dst, Register src) { // movzxb
4109 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true, true /* is_map1 */);
4110 emit_opcode_prefix_and_encoding((unsigned char)0xB6, 0xC0, encode);
4111 }
4112
4113 void Assembler::movzwl(Register dst, Address src) { // movzxw
4114 InstructionMark im(this);
4115 prefix(src, dst, false, true /* is_map1 */);
4116 emit_int8((unsigned char)0xB7);
4117 emit_operand(dst, src, 0);
4118 }
4119
4120 void Assembler::movzwl(Register dst, Register src) { // movzxw
4121 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
4122 emit_opcode_prefix_and_encoding((unsigned char)0xB7, 0xC0, encode);
4123 }
4124
4125 void Assembler::mull(Address src) {
4126 InstructionMark im(this);
4127 prefix(src);
4128 emit_int8((unsigned char)0xF7);
4129 emit_operand(rsp, src, 0);
4130 }
4131
4132 void Assembler::emull(Address src, bool no_flags) {
4133 InstructionMark im(this);
4134 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4135 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
4136 eevex_prefix_nf(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4137 emit_int8((unsigned char)0xF7);
4138 emit_operand(rsp, src, 0);
4139 }
4140
4141 void Assembler::mull(Register src) {
4142 int encode = prefix_and_encode(src->encoding());
4143 emit_int16((unsigned char)0xF7, (0xE0 | encode));
4144 }
4145
4146 void Assembler::emull(Register src, bool no_flags) {
4147 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4148 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4149 emit_int16((unsigned char)0xF7, (0xE0 | encode));
4150 }
4151
4152 void Assembler::mulsd(XMMRegister dst, Address src) {
4153 InstructionMark im(this);
4154 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4155 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4156 attributes.set_rex_vex_w_reverted();
4157 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4158 emit_int8(0x59);
4159 emit_operand(dst, src, 0);
4160 }
4161
4162 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
4163 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4164 attributes.set_rex_vex_w_reverted();
4165 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4166 emit_int16(0x59, (0xC0 | encode));
4167 }
4168
4169 void Assembler::mulss(XMMRegister dst, Address src) {
4170 InstructionMark im(this);
4171 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4172 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4173 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4174 emit_int8(0x59);
4175 emit_operand(dst, src, 0);
4176 }
4177
4178 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
4179 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4180 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4181 emit_int16(0x59, (0xC0 | encode));
4182 }
4183
4184 void Assembler::negl(Register dst) {
4185 int encode = prefix_and_encode(dst->encoding());
4186 emit_int16((unsigned char)0xF7, (0xD8 | encode));
4187 }
4188
4189 void Assembler::enegl(Register dst, Register src, bool no_flags) {
4190 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4191 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4192 emit_int16((unsigned char)0xF7, (0xD8 | encode));
4193 }
4194
4195 void Assembler::negl(Address dst) {
4196 InstructionMark im(this);
4197 prefix(dst);
4198 emit_int8((unsigned char)0xF7);
4199 emit_operand(as_Register(3), dst, 0);
4200 }
4201
4202 void Assembler::enegl(Register dst, Address src, bool no_flags) {
4203 InstructionMark im(this);
4204 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4205 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
4206 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4207 emit_int8((unsigned char)0xF7);
4208 emit_operand(as_Register(3), src, 0);
4209 }
4210
4211 void Assembler::nop(uint i) {
4212 #ifdef ASSERT
4213 assert(i > 0, " ");
4214 // The fancy nops aren't currently recognized by debuggers making it a
4215 // pain to disassemble code while debugging. If asserts are on clearly
4216 // speed is not an issue so simply use the single byte traditional nop
4217 // to do alignment.
4218
4219 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
4220 return;
4221
4222 #endif // ASSERT
4223
4224 if (UseAddressNop && VM_Version::is_intel()) {
4225 //
4226 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
4227 // 1: 0x90
4228 // 2: 0x66 0x90
4229 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
4230 // 4: 0x0F 0x1F 0x40 0x00
4231 // 5: 0x0F 0x1F 0x44 0x00 0x00
4232 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
4233 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
4234 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4235 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4236 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4237 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4238
4239 // The rest coding is Intel specific - don't use consecutive address nops
4240
4241 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4242 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4243 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4244 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4245
4246 while(i >= 15) {
4247 // For Intel don't generate consecutive address nops (mix with regular nops)
4248 i -= 15;
4249 emit_int24(0x66, 0x66, 0x66);
4250 addr_nop_8();
4251 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4252 }
4253 switch (i) {
4254 case 14:
4255 emit_int8(0x66); // size prefix
4256 case 13:
4257 emit_int8(0x66); // size prefix
4258 case 12:
4259 addr_nop_8();
4260 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4261 break;
4262 case 11:
4263 emit_int8(0x66); // size prefix
4264 case 10:
4265 emit_int8(0x66); // size prefix
4266 case 9:
4267 emit_int8(0x66); // size prefix
4268 case 8:
4269 addr_nop_8();
4270 break;
4271 case 7:
4272 addr_nop_7();
4273 break;
4274 case 6:
4275 emit_int8(0x66); // size prefix
4276 case 5:
4277 addr_nop_5();
4278 break;
4279 case 4:
4280 addr_nop_4();
4281 break;
4282 case 3:
4283 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
4284 emit_int8(0x66); // size prefix
4285 case 2:
4286 emit_int8(0x66); // size prefix
4287 case 1:
4288 emit_int8((unsigned char)0x90);
4289 // nop
4290 break;
4291 default:
4292 assert(i == 0, " ");
4293 }
4294 return;
4295 }
4296 if (UseAddressNop && VM_Version::is_amd_family()) {
4297 //
4298 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
4299 // 1: 0x90
4300 // 2: 0x66 0x90
4301 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
4302 // 4: 0x0F 0x1F 0x40 0x00
4303 // 5: 0x0F 0x1F 0x44 0x00 0x00
4304 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
4305 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
4306 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4307 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4308 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4309 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4310
4311 // The rest coding is AMD specific - use consecutive address nops
4312
4313 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
4314 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
4315 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
4316 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
4317 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4318 // Size prefixes (0x66) are added for larger sizes
4319
4320 while(i >= 22) {
4321 i -= 11;
4322 emit_int24(0x66, 0x66, 0x66);
4323 addr_nop_8();
4324 }
4325 // Generate first nop for size between 21-12
4326 switch (i) {
4327 case 21:
4328 i -= 1;
4329 emit_int8(0x66); // size prefix
4330 case 20:
4331 case 19:
4332 i -= 1;
4333 emit_int8(0x66); // size prefix
4334 case 18:
4335 case 17:
4336 i -= 1;
4337 emit_int8(0x66); // size prefix
4338 case 16:
4339 case 15:
4340 i -= 8;
4341 addr_nop_8();
4342 break;
4343 case 14:
4344 case 13:
4345 i -= 7;
4346 addr_nop_7();
4347 break;
4348 case 12:
4349 i -= 6;
4350 emit_int8(0x66); // size prefix
4351 addr_nop_5();
4352 break;
4353 default:
4354 assert(i < 12, " ");
4355 }
4356
4357 // Generate second nop for size between 11-1
4358 switch (i) {
4359 case 11:
4360 emit_int8(0x66); // size prefix
4361 case 10:
4362 emit_int8(0x66); // size prefix
4363 case 9:
4364 emit_int8(0x66); // size prefix
4365 case 8:
4366 addr_nop_8();
4367 break;
4368 case 7:
4369 addr_nop_7();
4370 break;
4371 case 6:
4372 emit_int8(0x66); // size prefix
4373 case 5:
4374 addr_nop_5();
4375 break;
4376 case 4:
4377 addr_nop_4();
4378 break;
4379 case 3:
4380 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
4381 emit_int8(0x66); // size prefix
4382 case 2:
4383 emit_int8(0x66); // size prefix
4384 case 1:
4385 emit_int8((unsigned char)0x90);
4386 // nop
4387 break;
4388 default:
4389 assert(i == 0, " ");
4390 }
4391 return;
4392 }
4393
4394 if (UseAddressNop && VM_Version::is_zx()) {
4395 //
4396 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
4397 // 1: 0x90
4398 // 2: 0x66 0x90
4399 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
4400 // 4: 0x0F 0x1F 0x40 0x00
4401 // 5: 0x0F 0x1F 0x44 0x00 0x00
4402 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
4403 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
4404 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4405 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4406 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4407 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
4408
4409 // The rest coding is ZX specific - don't use consecutive address nops
4410
4411 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4412 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4413 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4414 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
4415
4416 while (i >= 15) {
4417 // For ZX don't generate consecutive address nops (mix with regular nops)
4418 i -= 15;
4419 emit_int24(0x66, 0x66, 0x66);
4420 addr_nop_8();
4421 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4422 }
4423 switch (i) {
4424 case 14:
4425 emit_int8(0x66); // size prefix
4426 case 13:
4427 emit_int8(0x66); // size prefix
4428 case 12:
4429 addr_nop_8();
4430 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4431 break;
4432 case 11:
4433 emit_int8(0x66); // size prefix
4434 case 10:
4435 emit_int8(0x66); // size prefix
4436 case 9:
4437 emit_int8(0x66); // size prefix
4438 case 8:
4439 addr_nop_8();
4440 break;
4441 case 7:
4442 addr_nop_7();
4443 break;
4444 case 6:
4445 emit_int8(0x66); // size prefix
4446 case 5:
4447 addr_nop_5();
4448 break;
4449 case 4:
4450 addr_nop_4();
4451 break;
4452 case 3:
4453 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
4454 emit_int8(0x66); // size prefix
4455 case 2:
4456 emit_int8(0x66); // size prefix
4457 case 1:
4458 emit_int8((unsigned char)0x90);
4459 // nop
4460 break;
4461 default:
4462 assert(i == 0, " ");
4463 }
4464 return;
4465 }
4466
4467 // Using nops with size prefixes "0x66 0x90".
4468 // From AMD Optimization Guide:
4469 // 1: 0x90
4470 // 2: 0x66 0x90
4471 // 3: 0x66 0x66 0x90
4472 // 4: 0x66 0x66 0x66 0x90
4473 // 5: 0x66 0x66 0x90 0x66 0x90
4474 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
4475 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
4476 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
4477 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
4478 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
4479 //
4480 while (i > 12) {
4481 i -= 4;
4482 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4483 }
4484 // 1 - 12 nops
4485 if (i > 8) {
4486 if (i > 9) {
4487 i -= 1;
4488 emit_int8(0x66);
4489 }
4490 i -= 3;
4491 emit_int24(0x66, 0x66, (unsigned char)0x90);
4492 }
4493 // 1 - 8 nops
4494 if (i > 4) {
4495 if (i > 6) {
4496 i -= 1;
4497 emit_int8(0x66);
4498 }
4499 i -= 3;
4500 emit_int24(0x66, 0x66, (unsigned char)0x90);
4501 }
4502 switch (i) {
4503 case 4:
4504 emit_int8(0x66);
4505 case 3:
4506 emit_int8(0x66);
4507 case 2:
4508 emit_int8(0x66);
4509 case 1:
4510 emit_int8((unsigned char)0x90);
4511 break;
4512 default:
4513 assert(i == 0, " ");
4514 }
4515 }
4516
4517 void Assembler::notl(Register dst) {
4518 int encode = prefix_and_encode(dst->encoding());
4519 emit_int16((unsigned char)0xF7, (0xD0 | encode));
4520 }
4521
4522 void Assembler::enotl(Register dst, Register src) {
4523 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4524 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes);
4525 emit_int16((unsigned char)0xF7, (0xD0 | encode));
4526 }
4527
4528 void Assembler::eorw(Register dst, Register src1, Register src2, bool no_flags) {
4529 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
4530 }
4531
4532 void Assembler::orl(Address dst, int32_t imm32) {
4533 InstructionMark im(this);
4534 prefix(dst);
4535 emit_arith_operand(0x81, rcx, dst, imm32);
4536 }
4537
4538 void Assembler::eorl(Register dst, Address src, int32_t imm32, bool no_flags) {
4539 InstructionMark im(this);
4540 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4541 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
4542 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4543 emit_arith_operand(0x81, rcx, src, imm32);
4544 }
4545
4546 void Assembler::orl(Register dst, int32_t imm32) {
4547 prefix(dst);
4548 emit_arith(0x81, 0xC8, dst, imm32);
4549 }
4550
4551 void Assembler::eorl(Register dst, Register src, int32_t imm32, bool no_flags) {
4552 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xC8, no_flags);
4553 }
4554
4555 void Assembler::orl(Register dst, Address src) {
4556 InstructionMark im(this);
4557 prefix(src, dst);
4558 emit_int8(0x0B);
4559 emit_operand(dst, src, 0);
4560 }
4561
4562 void Assembler::eorl(Register dst, Register src1, Address src2, bool no_flags) {
4563 InstructionMark im(this);
4564 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, no_flags);
4565 }
4566
4567 void Assembler::orl(Register dst, Register src) {
4568 (void) prefix_and_encode(dst->encoding(), src->encoding());
4569 emit_arith(0x0B, 0xC0, dst, src);
4570 }
4571
4572 void Assembler::eorl(Register dst, Register src1, Register src2, bool no_flags) {
4573 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
4574 }
4575
4576 void Assembler::orl(Address dst, Register src) {
4577 InstructionMark im(this);
4578 prefix(dst, src);
4579 emit_int8(0x09);
4580 emit_operand(src, dst, 0);
4581 }
4582
4583 void Assembler::eorl(Register dst, Address src1, Register src2, bool no_flags) {
4584 InstructionMark im(this);
4585 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
4586 }
4587
4588 void Assembler::orb(Address dst, int imm8) {
4589 InstructionMark im(this);
4590 prefix(dst);
4591 emit_int8((unsigned char)0x80);
4592 emit_operand(rcx, dst, 1);
4593 emit_int8(imm8);
4594 }
4595
4596 void Assembler::eorb(Register dst, Address src, int imm8, bool no_flags) {
4597 InstructionMark im(this);
4598 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4599 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit);
4600 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
4601 emit_int8((unsigned char)0x80);
4602 emit_operand(rcx, src, 1);
4603 emit_int8(imm8);
4604 }
4605
4606 void Assembler::orb(Address dst, Register src) {
4607 InstructionMark im(this);
4608 prefix(dst, src, true);
4609 emit_int8(0x08);
4610 emit_operand(src, dst, 0);
4611 }
4612
4613 void Assembler::eorb(Register dst, Address src1, Register src2, bool no_flags) {
4614 InstructionMark im(this);
4615 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x08, no_flags, false /* is_map1 */, true /* is_commutative */);
4616 }
4617
4618 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
4619 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4620 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4621 emit_int16(0x63, (0xC0 | encode));
4622 }
4623
4624 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4625 assert(UseAVX > 0, "some form of AVX must be enabled");
4626 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4627 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4628 emit_int16(0x63, (0xC0 | encode));
4629 }
4630
4631 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
4632 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4633 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4634 emit_int16(0x6B, (0xC0 | encode));
4635 }
4636
4637 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4638 assert(UseAVX > 0, "some form of AVX must be enabled");
4639 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4640 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4641 emit_int16(0x6B, (0xC0 | encode));
4642 }
4643
4644 void Assembler::packuswb(XMMRegister dst, Address src) {
4645 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4646 InstructionMark im(this);
4647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4648 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4649 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4650 emit_int8(0x67);
4651 emit_operand(dst, src, 0);
4652 }
4653
4654 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
4655 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4656 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4657 emit_int16(0x67, (0xC0 | encode));
4658 }
4659
4660 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4661 assert(UseAVX > 0, "some form of AVX must be enabled");
4662 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4663 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4664 emit_int16(0x67, (0xC0 | encode));
4665 }
4666
4667 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
4668 assert(VM_Version::supports_sse4_1(), "");
4669 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4670 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4671 emit_int16(0x2B, (0xC0 | encode));
4672 }
4673
4674 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4675 assert(UseAVX > 0, "some form of AVX must be enabled");
4676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4677 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4678 emit_int16(0x2B, (0xC0 | encode));
4679 }
4680
4681 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4682 assert(VM_Version::supports_avx2(), "");
4683 assert(vector_len != AVX_128bit, "");
4684 // VEX.256.66.0F3A.W1 00 /r ib
4685 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4686 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4687 emit_int24(0x00, (0xC0 | encode), imm8);
4688 }
4689
4690 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4691 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
4692 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported");
4693 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4694 attributes.set_is_evex_instruction();
4695 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4696 emit_int16(0x36, (0xC0 | encode));
4697 }
4698
4699 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4700 assert(VM_Version::supports_avx512_vbmi(), "");
4701 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4702 attributes.set_is_evex_instruction();
4703 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4704 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4705 }
4706
4707 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4708 assert(VM_Version::supports_avx512_vbmi(), "");
4709 InstructionMark im(this);
4710 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4711 attributes.set_is_evex_instruction();
4712 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4713 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4714 emit_int8((unsigned char)0x8D);
4715 emit_operand(dst, src, 0);
4716 }
4717
4718 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4719 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
4720 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
4721 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
4722 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4723 attributes.set_is_evex_instruction();
4724 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4725 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4726 }
4727
4728 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4729 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
4730 (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
4731 // VEX.NDS.256.66.0F38.W0 36 /r
4732 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4733 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4734 emit_int16(0x36, (0xC0 | encode));
4735 }
4736
4737 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4738 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
4739 (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
4740 // VEX.NDS.256.66.0F38.W0 36 /r
4741 InstructionMark im(this);
4742 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
4744 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4745 emit_int8(0x36);
4746 emit_operand(dst, src, 0);
4747 }
4748
4749 void Assembler::vpermps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4750 assert((vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
4751 (vector_len == AVX_512bit && VM_Version::supports_evex()), "");
4752 // VEX.NDS.XXX.66.0F38.W0 16 /r
4753 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4754 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4755 emit_int16(0x16, (0xC0 | encode));
4756 }
4757
4758 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4759 assert(VM_Version::supports_avx2(), "");
4760 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4761 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4762 emit_int24(0x46, (0xC0 | encode), imm8);
4763 }
4764
4765 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4766 assert(VM_Version::supports_avx(), "");
4767 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4769 emit_int24(0x06, (0xC0 | encode), imm8);
4770 }
4771
4772 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4773 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4774 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4775 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4776 emit_int24(0x04, (0xC0 | encode), imm8);
4777 }
4778
4779 void Assembler::vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4780 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4781 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4782 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4783 emit_int16(0x0C, (0xC0 | encode));
4784 }
4785
4786 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4787 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4788 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
4789 attributes.set_rex_vex_w_reverted();
4790 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4791 emit_int24(0x05, (0xC0 | encode), imm8);
4792 }
4793
4794 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4795 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4796 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
4797 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4798 emit_int24(0x01, (0xC0 | encode), imm8);
4799 }
4800
4801 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
4802 assert(VM_Version::supports_avx512_vbmi(), "");
4803 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4804 attributes.set_is_evex_instruction();
4805 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4806 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
4807 }
4808
4809 void Assembler::pause() {
4810 emit_int16((unsigned char)0xF3, (unsigned char)0x90);
4811 }
4812
4813 void Assembler::ud2() {
4814 emit_int16(0x0F, 0x0B);
4815 }
4816
4817 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
4818 assert(VM_Version::supports_sse4_2(), "");
4819 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs as BASE or INDEX of address operand");
4820 InstructionMark im(this);
4821 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4822 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4823 emit_int8(0x61);
4824 emit_operand(dst, src, 1);
4825 emit_int8(imm8);
4826 }
4827
4828 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
4829 assert(VM_Version::supports_sse4_2(), "");
4830 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4831 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4832 emit_int24(0x61, (0xC0 | encode), imm8);
4833 }
4834
4835 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4836 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
4837 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4838 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4839 emit_int16(0x74, (0xC0 | encode));
4840 }
4841
4842 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4843 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4844 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4845 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4846 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4847 emit_int16(cond_encoding, (0xC0 | encode));
4848 }
4849
4850 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4851 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4852 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4853 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4854 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4855 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4856 emit_int16(0x74, (0xC0 | encode));
4857 }
4858
4859 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
4860 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4861 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4862 InstructionMark im(this);
4863 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4864 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4865 emit_int8(0x74);
4866 emit_operand(dst, src2, 0);
4867 }
4868
4869 // In this context, kdst is written the mask used to process the equal components
4870 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4871 assert(VM_Version::supports_avx512bw(), "");
4872 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4873 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4874 attributes.set_is_evex_instruction();
4875 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4876 emit_int16(0x74, (0xC0 | encode));
4877 }
4878
4879 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4880 assert(VM_Version::supports_avx512bw(), "");
4881 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4882 InstructionMark im(this);
4883 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4884 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4885 attributes.set_is_evex_instruction();
4886 int dst_enc = kdst->encoding();
4887 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4888 emit_int8(0x64);
4889 emit_operand(as_Register(dst_enc), src, 0);
4890 }
4891
4892 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4893 assert(VM_Version::supports_avx512bw(), "");
4894 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4895 InstructionMark im(this);
4896 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4897 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4898 attributes.reset_is_clear_context();
4899 attributes.set_embedded_opmask_register_specifier(mask);
4900 attributes.set_is_evex_instruction();
4901 int dst_enc = kdst->encoding();
4902 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4903 emit_int8(0x64);
4904 emit_operand(as_Register(dst_enc), src, 0);
4905 }
4906
4907 void Assembler::evpcmpub(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4908 assert(VM_Version::supports_avx512bw(), "");
4909 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4910 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4911 attributes.set_is_evex_instruction();
4912 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4913 emit_int24(0x3E, (0xC0 | encode), vcc);
4914 }
4915
4916 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4917 assert(VM_Version::supports_avx512bw(), "");
4918 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4919 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4920 attributes.set_is_evex_instruction();
4921 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4922 emit_int24(0x3E, (0xC0 | encode), vcc);
4923 }
4924
4925 void Assembler::evpcmpud(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4926 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4927 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4928 attributes.set_is_evex_instruction();
4929 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4930 emit_int24(0x1E, (0xC0 | encode), vcc);
4931 }
4932
4933 void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4934 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4935 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4936 attributes.set_is_evex_instruction();
4937 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4938 emit_int24(0x1E, (0xC0 | encode), vcc);
4939 }
4940
4941 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
4942 assert(VM_Version::supports_avx512bw(), "");
4943 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4944 InstructionMark im(this);
4945 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4946 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4947 attributes.set_is_evex_instruction();
4948 int dst_enc = kdst->encoding();
4949 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4950 emit_int8(0x3E);
4951 emit_operand(as_Register(dst_enc), src, 1);
4952 emit_int8(vcc);
4953 }
4954
4955 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4956 assert(VM_Version::supports_avx512bw(), "");
4957 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4958 InstructionMark im(this);
4959 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4960 attributes.set_is_evex_instruction();
4961 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4962 int dst_enc = kdst->encoding();
4963 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4964 emit_int8(0x74);
4965 emit_operand(as_Register(dst_enc), src, 0);
4966 }
4967
4968 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4969 assert(VM_Version::supports_avx512bw(), "");
4970 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
4971 InstructionMark im(this);
4972 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4973 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4974 attributes.reset_is_clear_context();
4975 attributes.set_embedded_opmask_register_specifier(mask);
4976 attributes.set_is_evex_instruction();
4977 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4978 emit_int8(0x74);
4979 emit_operand(as_Register(kdst->encoding()), src, 0);
4980 }
4981
4982 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4983 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
4984 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4985 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4986 emit_int16(0x75, (0xC0 | encode));
4987 }
4988
4989 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4990 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4991 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4992 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4993 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4994 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4995 emit_int16(0x75, (0xC0 | encode));
4996 }
4997
4998 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4999 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5000 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
5001 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
5002 InstructionMark im(this);
5003 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5004 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5005 emit_int8(0x75);
5006 emit_operand(dst, src, 0);
5007 }
5008
5009 // In this context, kdst is written the mask used to process the equal components
5010 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
5011 assert(VM_Version::supports_avx512bw(), "");
5012 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5013 attributes.set_is_evex_instruction();
5014 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5015 emit_int16(0x75, (0xC0 | encode));
5016 }
5017
5018 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
5019 assert(VM_Version::supports_avx512bw(), "");
5020 InstructionMark im(this);
5021 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5022 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5023 attributes.set_is_evex_instruction();
5024 int dst_enc = kdst->encoding();
5025 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5026 emit_int8(0x75);
5027 emit_operand(as_Register(dst_enc), src, 0);
5028 }
5029
5030 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
5031 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
5032 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5033 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5034 emit_int16(0x76, (0xC0 | encode));
5035 }
5036
5037 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
5038 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5039 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
5040 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
5041 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5043 emit_int16(0x76, (0xC0 | encode));
5044 }
5045
5046 // In this context, kdst is written the mask used to process the equal components
5047 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
5048 assert(VM_Version::supports_evex(), "");
5049 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5050 attributes.set_is_evex_instruction();
5051 attributes.reset_is_clear_context();
5052 attributes.set_embedded_opmask_register_specifier(mask);
5053 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5054 emit_int16(0x76, (0xC0 | encode));
5055 }
5056
5057 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
5058 assert(VM_Version::supports_evex(), "");
5059 InstructionMark im(this);
5060 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5061 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
5062 attributes.set_is_evex_instruction();
5063 attributes.reset_is_clear_context();
5064 attributes.set_embedded_opmask_register_specifier(mask);
5065 int dst_enc = kdst->encoding();
5066 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5067 emit_int8(0x76);
5068 emit_operand(as_Register(dst_enc), src, 0);
5069 }
5070
5071 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
5072 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
5073 assert(VM_Version::supports_sse4_1(), "");
5074 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5075 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5076 emit_int16(0x29, (0xC0 | encode));
5077 }
5078
5079 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
5080 assert(VM_Version::supports_evex(), "");
5081 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5082 attributes.set_is_evex_instruction();
5083 attributes.reset_is_clear_context();
5084 attributes.set_embedded_opmask_register_specifier(mask);
5085 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5086 emit_int16(0x29, (0xC0 | encode));
5087 }
5088
5089 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
5090 assert(VM_Version::supports_avx(), "");
5091 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5092 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5093 emit_int16(cond_encoding, (0xC0 | encode));
5094 }
5095
5096 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
5097 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5098 assert(VM_Version::supports_avx(), "");
5099 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5100 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5101 emit_int16(0x29, (0xC0 | encode));
5102 }
5103
5104 // In this context, kdst is written the mask used to process the equal components
5105 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
5106 assert(VM_Version::supports_evex(), "");
5107 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5108 attributes.reset_is_clear_context();
5109 attributes.set_is_evex_instruction();
5110 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5111 emit_int16(0x29, (0xC0 | encode));
5112 }
5113
5114 // In this context, kdst is written the mask used to process the equal components
5115 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
5116 assert(VM_Version::supports_evex(), "");
5117 InstructionMark im(this);
5118 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5119 attributes.reset_is_clear_context();
5120 attributes.set_is_evex_instruction();
5121 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
5122 int dst_enc = kdst->encoding();
5123 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5124 emit_int8(0x29);
5125 emit_operand(as_Register(dst_enc), src, 0);
5126 }
5127
5128 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
5129 assert(VM_Version::supports_sse4_1(), "");
5130 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5131 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5132 emit_int16(0x37, (0xC0 | encode));
5133 }
5134
5135 void Assembler::pmovmskb(Register dst, XMMRegister src) {
5136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5137 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5138 emit_int16((unsigned char)0xD7, (0xC0 | encode));
5139 }
5140
5141 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
5142 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
5143 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), "");
5144 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5145 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5146 emit_int16((unsigned char)0xD7, (0xC0 | encode));
5147 }
5148
5149 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
5150 assert(VM_Version::supports_avx(), "");
5151 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5152 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5153 emit_int16(0x50, (0xC0 | encode));
5154 }
5155
5156 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
5157 assert(VM_Version::supports_avx(), "");
5158 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5159 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5160 emit_int16(0x50, (0xC0 | encode));
5161 }
5162
5163
5164 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
5165 assert(VM_Version::supports_sse4_1(), "");
5166 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5167 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5168 emit_int24(0x16, (0xC0 | encode), imm8);
5169 }
5170
5171 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
5172 assert(VM_Version::supports_sse4_1(), "");
5173 InstructionMark im(this);
5174 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5175 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5176 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5177 emit_int8(0x16);
5178 emit_operand(src, dst, 1);
5179 emit_int8(imm8);
5180 }
5181
5182 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
5183 assert(VM_Version::supports_sse4_1(), "");
5184 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5185 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5186 emit_int24(0x16, (0xC0 | encode), imm8);
5187 }
5188
5189 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
5190 assert(VM_Version::supports_sse4_1(), "");
5191 InstructionMark im(this);
5192 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5193 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5194 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5195 emit_int8(0x16);
5196 emit_operand(src, dst, 1);
5197 emit_int8(imm8);
5198 }
5199
5200 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
5201 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5202 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5203 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
5204 }
5205
5206 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
5207 assert(VM_Version::supports_sse4_1(), "");
5208 InstructionMark im(this);
5209 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5210 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
5211 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5212 emit_int8(0x15);
5213 emit_operand(src, dst, 1);
5214 emit_int8(imm8);
5215 }
5216
5217 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
5218 assert(VM_Version::supports_sse4_1(), "");
5219 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5220 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5221 emit_int24(0x14, (0xC0 | encode), imm8);
5222 }
5223
5224 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
5225 assert(VM_Version::supports_sse4_1(), "");
5226 InstructionMark im(this);
5227 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5228 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
5229 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5230 emit_int8(0x14);
5231 emit_operand(src, dst, 1);
5232 emit_int8(imm8);
5233 }
5234
5235 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
5236 assert(VM_Version::supports_sse4_1(), "");
5237 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5238 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5239 emit_int24(0x22, (0xC0 | encode), imm8);
5240 }
5241
5242 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
5243 assert(VM_Version::supports_sse4_1(), "");
5244 InstructionMark im(this);
5245 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5246 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5247 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5248 emit_int8(0x22);
5249 emit_operand(dst, src, 1);
5250 emit_int8(imm8);
5251 }
5252
5253 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
5254 assert(VM_Version::supports_avx(), "");
5255 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5256 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5257 emit_int24(0x22, (0xC0 | encode), imm8);
5258 }
5259
5260 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
5261 assert(VM_Version::supports_sse4_1(), "");
5262 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5263 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5264 emit_int24(0x22, (0xC0 | encode), imm8);
5265 }
5266
5267 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
5268 assert(VM_Version::supports_sse4_1(), "");
5269 InstructionMark im(this);
5270 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5271 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5272 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5273 emit_int8(0x22);
5274 emit_operand(dst, src, 1);
5275 emit_int8(imm8);
5276 }
5277
5278 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
5279 assert(VM_Version::supports_avx(), "");
5280 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
5281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5282 emit_int24(0x22, (0xC0 | encode), imm8);
5283 }
5284
5285 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
5286 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5287 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
5288 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
5289 }
5290
5291 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
5292 InstructionMark im(this);
5293 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5294 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
5295 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5296 emit_int8((unsigned char)0xC4);
5297 emit_operand(dst, src, 1);
5298 emit_int8(imm8);
5299 }
5300
5301 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
5302 assert(VM_Version::supports_avx(), "");
5303 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5304 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
5305 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
5306 }
5307
5308 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
5309 assert(VM_Version::supports_sse4_1(), "");
5310 InstructionMark im(this);
5311 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5312 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
5313 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5314 emit_int8(0x20);
5315 emit_operand(dst, src, 1);
5316 emit_int8(imm8);
5317 }
5318
5319 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
5320 assert(VM_Version::supports_sse4_1(), "");
5321 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5322 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5323 emit_int24(0x20, (0xC0 | encode), imm8);
5324 }
5325
5326 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
5327 assert(VM_Version::supports_avx(), "");
5328 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
5329 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
5330 emit_int24(0x20, (0xC0 | encode), imm8);
5331 }
5332
5333 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
5334 assert(VM_Version::supports_sse4_1(), "");
5335 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5336 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5337 emit_int24(0x21, (0xC0 | encode), imm8);
5338 }
5339
5340 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
5341 assert(VM_Version::supports_avx(), "");
5342 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5343 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5344 emit_int24(0x21, (0xC0 | encode), imm8);
5345 }
5346
5347 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
5348 assert(VM_Version::supports_sse4_1(), "");
5349 InstructionMark im(this);
5350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5351 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
5352 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5353 emit_int8(0x30);
5354 emit_operand(dst, src, 0);
5355 }
5356
5357 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
5358 assert(VM_Version::supports_sse4_1(), "");
5359 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5360 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5361 emit_int16(0x30, (0xC0 | encode));
5362 }
5363
5364 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
5365 assert(VM_Version::supports_sse4_1(), "");
5366 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5367 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5368 emit_int16(0x20, (0xC0 | encode));
5369 }
5370
5371 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
5372 assert(VM_Version::supports_sse4_1(), "");
5373 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5374 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5375 emit_int16(0x35, (0xC0 | encode));
5376 }
5377
5378 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
5379 assert(VM_Version::supports_sse4_1(), "");
5380 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5381 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5382 emit_int16(0x21, (0xC0 | encode));
5383 }
5384
5385 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
5386 assert(VM_Version::supports_sse4_1(), "");
5387 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5388 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5389 emit_int16(0x31, (0xC0 | encode));
5390 }
5391
5392 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
5393 assert(VM_Version::supports_sse4_1(), "");
5394 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5395 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5396 emit_int16(0x22, (0xC0 | encode));
5397 }
5398
5399 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
5400 assert(VM_Version::supports_sse4_1(), "");
5401 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5402 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5403 emit_int16(0x23, (0xC0 | encode));
5404 }
5405
5406 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
5407 assert(VM_Version::supports_avx(), "");
5408 InstructionMark im(this);
5409 assert(dst != xnoreg, "sanity");
5410 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5411 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
5412 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5413 emit_int8(0x30);
5414 emit_operand(dst, src, 0);
5415 }
5416
5417 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
5418 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5419 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5420 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5421 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5422 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5423 emit_int16(0x30, (unsigned char) (0xC0 | encode));
5424 }
5425
5426 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
5427 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5428 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5429 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5430 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5431 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5432 emit_int16(0x20, (0xC0 | encode));
5433 }
5434
5435 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
5436 assert(VM_Version::supports_avx512vlbw(), "");
5437 assert(dst != xnoreg, "sanity");
5438 InstructionMark im(this);
5439 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5440 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
5441 attributes.set_embedded_opmask_register_specifier(mask);
5442 attributes.set_is_evex_instruction();
5443 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5444 emit_int8(0x30);
5445 emit_operand(dst, src, 0);
5446 }
5447
5448 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
5449 assert(VM_Version::supports_avx512vl(), "");
5450 assert(dst != xnoreg, "sanity");
5451 InstructionMark im(this);
5452 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5453 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
5454 attributes.set_embedded_opmask_register_specifier(mask);
5455 attributes.set_is_evex_instruction();
5456 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5457 emit_int8(0x31);
5458 emit_operand(dst, src, 0);
5459 }
5460
5461 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) {
5462 evpmovzxbd(dst, k0, src, vector_len);
5463 }
5464
5465 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
5466 assert(VM_Version::supports_evex(), "");
5467 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
5468 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5469 attributes.set_is_evex_instruction();
5470 attributes.set_embedded_opmask_register_specifier(mask);
5471 if (merge) {
5472 attributes.reset_is_clear_context();
5473 }
5474 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5475 emit_int16((unsigned char)0xDB, (0xC0 | encode));
5476 }
5477
5478 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
5479 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
5480 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5481 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5482 emit_int16(0x35, (0xC0 | encode));
5483 }
5484
5485 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
5486 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
5487 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5488 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5489 emit_int16(0x31, (0xC0 | encode));
5490 }
5491
5492 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
5493 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
5494 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5495 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5496 emit_int16(0x32, (0xC0 | encode));
5497 }
5498
5499 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
5500 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5501 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5502 VM_Version::supports_evex(), "");
5503 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5504 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5505 emit_int16(0x21, (0xC0 | encode));
5506 }
5507
5508 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
5509 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5510 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5511 VM_Version::supports_evex(), "");
5512 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5513 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5514 emit_int16(0x22, (0xC0 | encode));
5515 }
5516
5517 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
5518 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5519 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5520 VM_Version::supports_evex(), "");
5521 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5522 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5523 emit_int16(0x23, (0xC0 | encode));
5524 }
5525
5526 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
5527 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5528 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5529 VM_Version::supports_evex(), "");
5530 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5531 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5532 emit_int16(0x24, (0xC0 | encode));
5533 }
5534
5535 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
5536 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5537 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5538 VM_Version::supports_evex(), "");
5539 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5540 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5541 emit_int16(0x25, (0xC0 | encode));
5542 }
5543
5544 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
5545 assert(VM_Version::supports_avx512vlbw(), "");
5546 assert(src != xnoreg, "sanity");
5547 InstructionMark im(this);
5548 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5549 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
5550 attributes.set_is_evex_instruction();
5551 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
5552 emit_int8(0x30);
5553 emit_operand(src, dst, 0);
5554 }
5555
5556 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
5557 assert(VM_Version::supports_avx512vlbw(), "");
5558 assert(src != xnoreg, "sanity");
5559 InstructionMark im(this);
5560 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5561 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
5562 attributes.reset_is_clear_context();
5563 attributes.set_embedded_opmask_register_specifier(mask);
5564 attributes.set_is_evex_instruction();
5565 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
5566 emit_int8(0x30);
5567 emit_operand(src, dst, 0);
5568 }
5569
5570 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
5571 assert(VM_Version::supports_evex(), "");
5572 assert(src != xnoreg, "sanity");
5573 InstructionMark im(this);
5574 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5575 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
5576 attributes.set_is_evex_instruction();
5577 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
5578 emit_int8(0x31);
5579 emit_operand(src, dst, 0);
5580 }
5581
5582 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
5583 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5584 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5585 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
5586 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5587 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5588 emit_int16(0x33, (0xC0 | encode));
5589 }
5590
5591 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
5592 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5593 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5594 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
5595 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5596 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5597 emit_int16(0x34, (0xC0 | encode));
5598 }
5599
5600 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
5601 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5602 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5603 emit_int16((unsigned char)0xF5, (0xC0 | encode));
5604 }
5605
5606 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5607 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5608 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5609 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
5610 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5611 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5612 emit_int16((unsigned char)0xF5, (0xC0 | encode));
5613 }
5614
5615 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5616 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5617 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5618 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5619 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5620 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5621 emit_int16(0x04, (0xC0 | encode));
5622 }
5623
5624 void Assembler::vpmadd52luq(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5625 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), "");
5626
5627 InstructionMark im(this);
5628 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5629
5630 if (VM_Version::supports_avx512ifma()) {
5631 attributes.set_is_evex_instruction();
5632 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
5633 }
5634 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5635 emit_int8((unsigned char)0xB4);
5636 emit_operand(dst, src2, 0);
5637 }
5638
5639 void Assembler::vpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5640 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), "");
5641
5642 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5643
5644 if (VM_Version::supports_avx512ifma()) {
5645 attributes.set_is_evex_instruction();
5646 }
5647 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5648 emit_int16((unsigned char)0xB4, (0xC0 | encode));
5649 }
5650
5651 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5652 evpmadd52luq(dst, k0, src1, src2, false, vector_len);
5653 }
5654
5655 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5656 assert(VM_Version::supports_avx512ifma(), "");
5657 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5658 attributes.set_is_evex_instruction();
5659 attributes.set_embedded_opmask_register_specifier(mask);
5660 if (merge) {
5661 attributes.reset_is_clear_context();
5662 }
5663
5664 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5665 emit_int16((unsigned char)0xB4, (0xC0 | encode));
5666 }
5667
5668 void Assembler::vpmadd52huq(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5669 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), "");
5670
5671 InstructionMark im(this);
5672 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5673
5674 if (VM_Version::supports_avx512ifma()) {
5675 attributes.set_is_evex_instruction();
5676 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
5677 }
5678 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5679 emit_int8((unsigned char)0xB5);
5680 emit_operand(dst, src2, 0);
5681 }
5682
5683 void Assembler::vpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5684 assert ((VM_Version::supports_avxifma() && vector_len <= AVX_256bit) || (VM_Version::supports_avx512ifma() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl())), "");
5685
5686 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5687
5688 if (VM_Version::supports_avx512ifma()) {
5689 attributes.set_is_evex_instruction();
5690 }
5691 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5692 emit_int16((unsigned char)0xB5, (0xC0 | encode));
5693 }
5694
5695 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5696 evpmadd52huq(dst, k0, src1, src2, false, vector_len);
5697 }
5698
5699 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5700 assert(VM_Version::supports_avx512ifma(), "");
5701 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5702 attributes.set_is_evex_instruction();
5703 attributes.set_embedded_opmask_register_specifier(mask);
5704 if (merge) {
5705 attributes.reset_is_clear_context();
5706 }
5707
5708 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5709 emit_int16((unsigned char)0xB5, (0xC0 | encode));
5710 }
5711
5712 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5713 assert(VM_Version::supports_evex(), "");
5714 assert(VM_Version::supports_avx512_vnni(), "must support vnni");
5715 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5716 attributes.set_is_evex_instruction();
5717 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5718 emit_int16(0x52, (0xC0 | encode));
5719 }
5720
5721 // generic
5722 void Assembler::pop(Register dst) {
5723 int encode = prefix_and_encode(dst->encoding());
5724 emit_int8(0x58 | encode);
5725 }
5726
5727 void Assembler::popcntl(Register dst, Address src) {
5728 assert(VM_Version::supports_popcnt(), "must support");
5729 InstructionMark im(this);
5730 emit_int8((unsigned char)0xF3);
5731 prefix(src, dst, false, true /* is_map1 */);
5732 emit_int8((unsigned char)0xB8);
5733 emit_operand(dst, src, 0);
5734 }
5735
5736 void Assembler::epopcntl(Register dst, Address src, bool no_flags) {
5737 assert(VM_Version::supports_popcnt(), "must support");
5738 InstructionMark im(this);
5739 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5740 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
5741 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
5742 emit_int8((unsigned char)0x88);
5743 emit_operand(dst, src, 0);
5744 }
5745
5746 void Assembler::popcntl(Register dst, Register src) {
5747 assert(VM_Version::supports_popcnt(), "must support");
5748 emit_int8((unsigned char)0xF3);
5749 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
5750 emit_opcode_prefix_and_encoding((unsigned char)0xB8, 0xC0, encode);
5751 }
5752
5753 void Assembler::epopcntl(Register dst, Register src, bool no_flags) {
5754 assert(VM_Version::supports_popcnt(), "must support");
5755 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5756 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
5757 emit_int16((unsigned char)0x88, (0xC0 | encode));
5758 }
5759
5760 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5761 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5762 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5763 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5764 attributes.set_embedded_opmask_register_specifier(mask);
5765 attributes.set_is_evex_instruction();
5766 if (merge) {
5767 attributes.reset_is_clear_context();
5768 }
5769 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5770 emit_int16(0x54, (0xC0 | encode));
5771 }
5772
5773 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5774 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5775 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5776 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5777 attributes.set_is_evex_instruction();
5778 attributes.set_embedded_opmask_register_specifier(mask);
5779 if (merge) {
5780 attributes.reset_is_clear_context();
5781 }
5782 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5783 emit_int16(0x54, (0xC0 | encode));
5784 }
5785
5786 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5787 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5788 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5789 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5790 attributes.set_is_evex_instruction();
5791 attributes.set_embedded_opmask_register_specifier(mask);
5792 if (merge) {
5793 attributes.reset_is_clear_context();
5794 }
5795 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5796 emit_int16(0x55, (0xC0 | encode));
5797 }
5798
5799 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5800 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5801 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5802 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5803 attributes.set_is_evex_instruction();
5804 attributes.set_embedded_opmask_register_specifier(mask);
5805 if (merge) {
5806 attributes.reset_is_clear_context();
5807 }
5808 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5809 emit_int16(0x55, (0xC0 | encode));
5810 }
5811
5812 void Assembler::popf() {
5813 emit_int8((unsigned char)0x9D);
5814 }
5815
5816 void Assembler::prefetchnta(Address src) {
5817 InstructionMark im(this);
5818 prefix(src, true /* is_map1 */);
5819 emit_int8(0x18);
5820 emit_operand(rax, src, 0); // 0, src
5821 }
5822
5823 void Assembler::prefetchr(Address src) {
5824 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5825 InstructionMark im(this);
5826 prefix(src, true /* is_map1 */);
5827 emit_int8(0x0D);
5828 emit_operand(rax, src, 0); // 0, src
5829 }
5830
5831 void Assembler::prefetcht0(Address src) {
5832 InstructionMark im(this);
5833 prefix(src, true /* is_map1 */);
5834 emit_int8(0x18);
5835 emit_operand(rcx, src, 0); // 1, src
5836 }
5837
5838 void Assembler::prefetcht1(Address src) {
5839 InstructionMark im(this);
5840 prefix(src, true /* is_map1 */);
5841 emit_int8(0x18);
5842 emit_operand(rdx, src, 0); // 2, src
5843 }
5844
5845 void Assembler::prefetcht2(Address src) {
5846 InstructionMark im(this);
5847 prefix(src, true /* is_map1 */);
5848 emit_int8(0x18);
5849 emit_operand(rbx, src, 0); // 3, src
5850 }
5851
5852 void Assembler::prefetchw(Address src) {
5853 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5854 InstructionMark im(this);
5855 prefix(src, true /* is_map1 */);
5856 emit_int8(0x0D);
5857 emit_operand(rcx, src, 0); // 1, src
5858 }
5859
5860 void Assembler::prefix(Prefix p) {
5861 emit_int8(p);
5862 }
5863
5864 void Assembler::prefix16(int prefix) {
5865 assert(UseAPX, "APX features not enabled");
5866 emit_int8((prefix & 0xff00) >> 8);
5867 emit_int8(prefix & 0xff);
5868 }
5869
5870 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
5871 assert(VM_Version::supports_ssse3(), "");
5872 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5873 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5874 emit_int16(0x00, (0xC0 | encode));
5875 }
5876
5877 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
5878 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
5879 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5880 attributes.set_is_evex_instruction();
5881 attributes.set_embedded_opmask_register_specifier(mask);
5882 if (merge) {
5883 attributes.reset_is_clear_context();
5884 }
5885 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5886 emit_int16(0x00, (0xC0 | encode));
5887 }
5888
5889 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5890 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5891 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5892 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5893 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5894 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5895 emit_int16(0x00, (0xC0 | encode));
5896 }
5897
5898 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5899 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5900 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5901 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5902 InstructionMark im(this);
5903 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5904 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5905 simd_prefix(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5906 emit_int8(0x00);
5907 emit_operand(dst, src, 0);
5908 }
5909
5910 void Assembler::pshufb(XMMRegister dst, Address src) {
5911 assert(VM_Version::supports_ssse3(), "");
5912 InstructionMark im(this);
5913 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5914 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5915 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5916 emit_int8(0x00);
5917 emit_operand(dst, src, 0);
5918 }
5919
5920 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
5921 assert(isByte(mode), "invalid value");
5922 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
5923 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5924 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5925 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5926 }
5927
5928 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5929 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5930 (vector_len == AVX_256bit? VM_Version::supports_avx2() :
5931 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
5932 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5933 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5934 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5935 }
5936
5937 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
5938 assert(isByte(mode), "invalid value");
5939 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5940 InstructionMark im(this);
5941 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
5943 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5944 emit_int8(0x70);
5945 emit_operand(dst, src, 1);
5946 emit_int8(mode & 0xFF);
5947 }
5948
5949 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
5950 assert(isByte(mode), "invalid value");
5951 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5952 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5953 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5954 }
5955
5956 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5957 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5958 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5959 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
5960 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5961 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5962 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5963 }
5964
5965 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5966 assert(isByte(mode), "invalid value");
5967 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5968 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5969 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5970 }
5971
5972 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
5973 assert(isByte(mode), "invalid value");
5974 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5975 InstructionMark im(this);
5976 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5977 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5978 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5979 emit_int8(0x70);
5980 emit_operand(dst, src, 1);
5981 emit_int8(mode & 0xFF);
5982 }
5983
5984 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5985 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5986 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5987 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
5988 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5989 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5990 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5991 }
5992
5993 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5994 assert(VM_Version::supports_evex(), "requires EVEX support");
5995 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
5996 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5997 attributes.set_is_evex_instruction();
5998 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5999 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
6000 }
6001
6002 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
6003 assert(isByte(imm8), "invalid value");
6004 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6005 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6006 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
6007 }
6008
6009 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
6010 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6011 attributes.set_rex_vex_w_reverted();
6012 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6013 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
6014 }
6015
6016 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
6017 assert(isByte(imm8), "invalid value");
6018 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6019 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6020 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
6021 }
6022
6023 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
6024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6025 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6026 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
6027 }
6028
6029 void Assembler::psrldq(XMMRegister dst, int shift) {
6030 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
6031 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6032 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6033 emit_int24(0x73, (0xC0 | encode), shift);
6034 }
6035
6036 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6037 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
6038 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
6039 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
6040 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6041 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6042 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
6043 }
6044
6045 void Assembler::pslldq(XMMRegister dst, int shift) {
6046 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
6047 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6048 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
6049 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6050 emit_int24(0x73, (0xC0 | encode), shift);
6051 }
6052
6053 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6054 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
6055 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
6056 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
6057 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6058 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6059 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
6060 }
6061
6062 void Assembler::ptest(XMMRegister dst, Address src) {
6063 assert(VM_Version::supports_sse4_1(), "");
6064 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
6065 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs");
6066 InstructionMark im(this);
6067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6068 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6069 emit_int8(0x17);
6070 emit_operand(dst, src, 0);
6071 }
6072
6073 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
6074 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
6075 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6076 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6077 emit_int8(0x17);
6078 emit_int8((0xC0 | encode));
6079 }
6080
6081 void Assembler::vptest(XMMRegister dst, Address src) {
6082 assert(VM_Version::supports_avx(), "");
6083 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs");
6084 InstructionMark im(this);
6085 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6086 assert(dst != xnoreg, "sanity");
6087 // swap src<->dst for encoding
6088 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6089 emit_int8(0x17);
6090 emit_operand(dst, src, 0);
6091 }
6092
6093 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
6094 assert(VM_Version::supports_avx(), "");
6095 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6096 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6097 emit_int16(0x17, (0xC0 | encode));
6098 }
6099
6100 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
6101 assert(VM_Version::supports_avx(), "");
6102 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6103 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6104 emit_int16(0x17, (0xC0 | encode));
6105 }
6106
6107 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) {
6108 assert(VM_Version::supports_avx(), "");
6109 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6110 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6111 emit_int16(0x0E, (0xC0 | encode));
6112 }
6113
6114 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6115 assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), "");
6116 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
6117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6118 attributes.set_is_evex_instruction();
6119 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6120 emit_int16(0x26, (0xC0 | encode));
6121 }
6122
6123 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6124 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
6125 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
6126 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6127 attributes.set_is_evex_instruction();
6128 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6129 emit_int16(0x27, (0xC0 | encode));
6130 }
6131
6132 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6133 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
6134 // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r
6135 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6136 attributes.set_is_evex_instruction();
6137 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
6138 emit_int16(0x27, (0xC0 | encode));
6139 }
6140
6141 void Assembler::punpcklbw(XMMRegister dst, Address src) {
6142 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
6143 InstructionMark im(this);
6144 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
6145 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
6146 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6147 emit_int8(0x60);
6148 emit_operand(dst, src, 0);
6149 }
6150
6151 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
6152 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
6153 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6154 emit_int16(0x60, (0xC0 | encode));
6155 }
6156
6157 void Assembler::punpckldq(XMMRegister dst, Address src) {
6158 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
6159 InstructionMark im(this);
6160 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6161 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
6162 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6163 emit_int8(0x62);
6164 emit_operand(dst, src, 0);
6165 }
6166
6167 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
6168 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6169 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6170 emit_int16(0x62, (0xC0 | encode));
6171 }
6172
6173 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
6174 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6175 attributes.set_rex_vex_w_reverted();
6176 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6177 emit_int16(0x6C, (0xC0 | encode));
6178 }
6179
6180 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6181 evpunpcklqdq(dst, k0, src1, src2, false, vector_len);
6182 }
6183
6184 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
6185 assert(VM_Version::supports_evex(), "requires AVX512F");
6186 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
6187 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6188 attributes.set_is_evex_instruction();
6189 attributes.set_embedded_opmask_register_specifier(mask);
6190 if (merge) {
6191 attributes.reset_is_clear_context();
6192 }
6193
6194 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6195 emit_int16(0x6C, (0xC0 | encode));
6196 }
6197
6198 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6199 evpunpckhqdq(dst, k0, src1, src2, false, vector_len);
6200 }
6201
6202 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
6203 assert(VM_Version::supports_evex(), "requires AVX512F");
6204 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
6205 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6206 attributes.set_is_evex_instruction();
6207 attributes.set_embedded_opmask_register_specifier(mask);
6208 if (merge) {
6209 attributes.reset_is_clear_context();
6210 }
6211
6212 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6213 emit_int16(0x6D, (0xC0 | encode));
6214 }
6215
6216 void Assembler::push2(Register src1, Register src2, bool with_ppx) {
6217 assert(VM_Version::supports_apx_f(), "requires APX");
6218 InstructionAttr attributes(0, /* rex_w */ with_ppx, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6219 /* EVEX.BASE */
6220 int src_enc = src1->encoding();
6221 /* EVEX.VVVV */
6222 int nds_enc = src2->encoding();
6223
6224 bool vex_b = (src_enc & 8) == 8;
6225 bool evex_v = (nds_enc >= 16);
6226 bool evex_b = (src_enc >= 16);
6227
6228 // EVEX.ND = 1;
6229 attributes.set_extended_context();
6230 attributes.set_is_evex_instruction();
6231 set_attributes(&attributes);
6232
6233 evex_prefix(0, vex_b, 0, 0, evex_b, evex_v, false /*eevex_x*/, nds_enc, VEX_SIMD_NONE, /* map4 */ VEX_OPCODE_0F_3C);
6234 emit_int16(0xFF, (0xC0 | (0x6 << 3) | (src_enc & 7)));
6235 }
6236
6237 void Assembler::pop2(Register src1, Register src2, bool with_ppx) {
6238 assert(VM_Version::supports_apx_f(), "requires APX");
6239 InstructionAttr attributes(0, /* rex_w */ with_ppx, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6240 /* EVEX.BASE */
6241 int src_enc = src1->encoding();
6242 /* EVEX.VVVV */
6243 int nds_enc = src2->encoding();
6244
6245 bool vex_b = (src_enc & 8) == 8;
6246 bool evex_v = (nds_enc >= 16);
6247 bool evex_b = (src_enc >= 16);
6248
6249 // EVEX.ND = 1;
6250 attributes.set_extended_context();
6251 attributes.set_is_evex_instruction();
6252 set_attributes(&attributes);
6253
6254 evex_prefix(0, vex_b, 0, 0, evex_b, evex_v, false /*eevex_x*/, nds_enc, VEX_SIMD_NONE, /* map4 */ VEX_OPCODE_0F_3C);
6255 emit_int16(0x8F, (0xC0 | (src_enc & 7)));
6256 }
6257
6258 void Assembler::push2p(Register src1, Register src2) {
6259 push2(src1, src2, true);
6260 }
6261
6262 void Assembler::pop2p(Register src1, Register src2) {
6263 pop2(src1, src2, true);
6264 }
6265
6266 void Assembler::pushp(Register src) {
6267 assert(VM_Version::supports_apx_f(), "requires APX");
6268 int encode = prefixq_and_encode_rex2(src->encoding());
6269 emit_int8(0x50 | encode);
6270 }
6271
6272 void Assembler::popp(Register dst) {
6273 assert(VM_Version::supports_apx_f(), "requires APX");
6274 int encode = prefixq_and_encode_rex2(dst->encoding());
6275 emit_int8((unsigned char)0x58 | encode);
6276 }
6277
6278 void Assembler::push(int32_t imm32) {
6279 // in 64bits we push 64bits onto the stack but only
6280 // take a 32bit immediate
6281 emit_int8(0x68);
6282 emit_int32(imm32);
6283 }
6284
6285 void Assembler::push(Register src) {
6286 int encode = prefix_and_encode(src->encoding());
6287 emit_int8(0x50 | encode);
6288 }
6289
6290 void Assembler::pushf() {
6291 emit_int8((unsigned char)0x9C);
6292 }
6293
6294 void Assembler::rcll(Register dst, int imm8) {
6295 assert(isShiftCount(imm8), "illegal shift count");
6296 int encode = prefix_and_encode(dst->encoding());
6297 if (imm8 == 1) {
6298 emit_int16((unsigned char)0xD1, (0xD0 | encode));
6299 } else {
6300 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
6301 }
6302 }
6303
6304 void Assembler::ercll(Register dst, Register src, int imm8) {
6305 assert(isShiftCount(imm8), "illegal shift count");
6306 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6307 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes);
6308 if (imm8 == 1) {
6309 emit_int16((unsigned char)0xD1, (0xD0 | encode));
6310 } else {
6311 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
6312 }
6313 }
6314
6315 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
6316 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6317 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6318 emit_int16(0x53, (0xC0 | encode));
6319 }
6320
6321 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
6322 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6323 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6324 emit_int16(0x53, (0xC0 | encode));
6325 }
6326
6327 void Assembler::rdtsc() {
6328 emit_int16(0x0F, 0x31);
6329 }
6330
6331 // copies data from [esi] to [edi] using rcx pointer sized words
6332 // generic
6333 void Assembler::rep_mov() {
6334 // REP
6335 // MOVSQ
6336 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);
6337 }
6338
6339 // sets rcx bytes with rax, value at [edi]
6340 void Assembler::rep_stosb() {
6341 // REP
6342 // STOSB
6343 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);
6344 }
6345
6346 // sets rcx pointer sized words with rax, value at [edi]
6347 // generic
6348 void Assembler::rep_stos() {
6349 // REP
6350 // STOSQ
6351 emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);
6352 }
6353
6354 // scans rcx pointer sized words at [edi] for occurrence of rax,
6355 // generic
6356 void Assembler::repne_scan() { // repne_scan
6357 // SCASQ
6358 emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);
6359 }
6360
6361 // scans rcx 4 byte words at [edi] for occurrence of rax,
6362 // generic
6363 void Assembler::repne_scanl() { // repne_scan
6364 // SCASL
6365 emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
6366 }
6367
6368 void Assembler::ret(int imm16) {
6369 if (imm16 == 0) {
6370 emit_int8((unsigned char)0xC3);
6371 } else {
6372 emit_int8((unsigned char)0xC2);
6373 emit_int16(imm16);
6374 }
6375 }
6376
6377 void Assembler::roll(Register dst, int imm8) {
6378 assert(isShiftCount(imm8), "illegal shift count");
6379 int encode = prefix_and_encode(dst->encoding());
6380 if (imm8 == 1) {
6381 emit_int16((unsigned char)0xD1, (0xC0 | encode));
6382 } else {
6383 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
6384 }
6385 }
6386
6387 void Assembler::eroll(Register dst, Register src, int imm8, bool no_flags) {
6388 assert(isShiftCount(imm8), "illegal shift count");
6389 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6390 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6391 if (imm8 == 1) {
6392 emit_int16((unsigned char)0xD1, (0xC0 | encode));
6393 } else {
6394 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
6395 }
6396 }
6397
6398 void Assembler::roll(Register dst) {
6399 int encode = prefix_and_encode(dst->encoding());
6400 emit_int16((unsigned char)0xD3, (0xC0 | encode));
6401 }
6402
6403 void Assembler::eroll(Register dst, Register src, bool no_flags) {
6404 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6405 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6406 emit_int16((unsigned char)0xD3, (0xC0 | encode));
6407 }
6408
6409 void Assembler::rorl(Register dst, int imm8) {
6410 assert(isShiftCount(imm8), "illegal shift count");
6411 int encode = prefix_and_encode(dst->encoding());
6412 if (imm8 == 1) {
6413 emit_int16((unsigned char)0xD1, (0xC8 | encode));
6414 } else {
6415 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
6416 }
6417 }
6418
6419 void Assembler::erorl(Register dst, Register src, int imm8, bool no_flags) {
6420 assert(isShiftCount(imm8), "illegal shift count");
6421 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6422 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6423 if (imm8 == 1) {
6424 emit_int16((unsigned char)0xD1, (0xC8 | encode));
6425 } else {
6426 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
6427 }
6428 }
6429
6430 void Assembler::rorl(Register dst) {
6431 int encode = prefix_and_encode(dst->encoding());
6432 emit_int16((unsigned char)0xD3, (0xC8 | encode));
6433 }
6434
6435 void Assembler::erorl(Register dst, Register src, bool no_flags) {
6436 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6437 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6438 emit_int16((unsigned char)0xD3, (0xC8 | encode));
6439 }
6440
6441 void Assembler::rorq(Register dst) {
6442 int encode = prefixq_and_encode(dst->encoding());
6443 emit_int16((unsigned char)0xD3, (0xC8 | encode));
6444 }
6445
6446 void Assembler::erorq(Register dst, Register src, bool no_flags) {
6447 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6448 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
6449 emit_int16((unsigned char)0xD3, (0xC8 | encode));
6450 }
6451
6452 void Assembler::rorq(Register dst, int imm8) {
6453 assert(isShiftCount(imm8 >> 1), "illegal shift count");
6454 int encode = prefixq_and_encode(dst->encoding());
6455 if (imm8 == 1) {
6456 emit_int16((unsigned char)0xD1, (0xC8 | encode));
6457 } else {
6458 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
6459 }
6460 }
6461
6462 void Assembler::erorq(Register dst, Register src, int imm8, bool no_flags) {
6463 assert(isShiftCount(imm8), "illegal shift count");
6464 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6465 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
6466 if (imm8 == 1) {
6467 emit_int16((unsigned char)0xD1, (0xC8 | encode));
6468 } else {
6469 emit_int24((unsigned char)0xC1, (0xC8 | encode), imm8);
6470 }
6471 }
6472
6473 void Assembler::rolq(Register dst) {
6474 int encode = prefixq_and_encode(dst->encoding());
6475 emit_int16((unsigned char)0xD3, (0xC0 | encode));
6476 }
6477
6478 void Assembler::erolq(Register dst, Register src, bool no_flags) {
6479 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6480 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
6481 emit_int16((unsigned char)0xD3, (0xC0 | encode));
6482 }
6483
6484 void Assembler::rolq(Register dst, int imm8) {
6485 assert(isShiftCount(imm8 >> 1), "illegal shift count");
6486 int encode = prefixq_and_encode(dst->encoding());
6487 if (imm8 == 1) {
6488 emit_int16((unsigned char)0xD1, (0xC0 | encode));
6489 } else {
6490 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
6491 }
6492 }
6493
6494 void Assembler::erolq(Register dst, Register src, int imm8, bool no_flags) {
6495 assert(isShiftCount(imm8), "illegal shift count");
6496 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6497 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
6498 if (imm8 == 1) {
6499 emit_int16((unsigned char)0xD1, (0xC0 | encode));
6500 } else {
6501 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
6502 }
6503 }
6504
6505 void Assembler::sall(Address dst, int imm8) {
6506 InstructionMark im(this);
6507 assert(isShiftCount(imm8), "illegal shift count");
6508 prefix(dst);
6509 if (imm8 == 1) {
6510 emit_int8((unsigned char)0xD1);
6511 emit_operand(as_Register(4), dst, 0);
6512 }
6513 else {
6514 emit_int8((unsigned char)0xC1);
6515 emit_operand(as_Register(4), dst, 1);
6516 emit_int8(imm8);
6517 }
6518 }
6519
6520 void Assembler::esall(Register dst, Address src, int imm8, bool no_flags) {
6521 InstructionMark im(this);
6522 assert(isShiftCount(imm8), "illegal shift count");
6523 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6524 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6525 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6526 if (imm8 == 1) {
6527 emit_int8((unsigned char)0xD1);
6528 emit_operand(as_Register(4), src, 0);
6529 }
6530 else {
6531 emit_int8((unsigned char)0xC1);
6532 emit_operand(as_Register(4), src, 1);
6533 emit_int8(imm8);
6534 }
6535 }
6536
6537 void Assembler::sall(Address dst) {
6538 InstructionMark im(this);
6539 prefix(dst);
6540 emit_int8((unsigned char)0xD3);
6541 emit_operand(as_Register(4), dst, 0);
6542 }
6543
6544 void Assembler::esall(Register dst, Address src, bool no_flags) {
6545 InstructionMark im(this);
6546 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6547 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6548 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6549 emit_int8((unsigned char)0xD3);
6550 emit_operand(as_Register(4), src, 0);
6551 }
6552
6553 void Assembler::sall(Register dst, int imm8) {
6554 assert(isShiftCount(imm8), "illegal shift count");
6555 int encode = prefix_and_encode(dst->encoding());
6556 if (imm8 == 1) {
6557 emit_int16((unsigned char)0xD1, (0xE0 | encode));
6558 } else {
6559 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
6560 }
6561 }
6562
6563 void Assembler::esall(Register dst, Register src, int imm8, bool no_flags) {
6564 assert(isShiftCount(imm8), "illegal shift count");
6565 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6566 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6567 if (imm8 == 1) {
6568 emit_int16((unsigned char)0xD1, (0xE0 | encode));
6569 } else {
6570 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
6571 }
6572 }
6573
6574 void Assembler::sall(Register dst) {
6575 int encode = prefix_and_encode(dst->encoding());
6576 emit_int16((unsigned char)0xD3, (0xE0 | encode));
6577 }
6578
6579 void Assembler::esall(Register dst, Register src, bool no_flags) {
6580 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6581 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6582 emit_int16((unsigned char)0xD3, (0xE0 | encode));
6583 }
6584
6585 void Assembler::sarl(Address dst, int imm8) {
6586 assert(isShiftCount(imm8), "illegal shift count");
6587 InstructionMark im(this);
6588 prefix(dst);
6589 if (imm8 == 1) {
6590 emit_int8((unsigned char)0xD1);
6591 emit_operand(as_Register(7), dst, 0);
6592 }
6593 else {
6594 emit_int8((unsigned char)0xC1);
6595 emit_operand(as_Register(7), dst, 1);
6596 emit_int8(imm8);
6597 }
6598 }
6599
6600 void Assembler::esarl(Register dst, Address src, int imm8, bool no_flags) {
6601 assert(isShiftCount(imm8), "illegal shift count");
6602 InstructionMark im(this);
6603 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6604 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6605 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6606 if (imm8 == 1) {
6607 emit_int8((unsigned char)0xD1);
6608 emit_operand(as_Register(7), src, 0);
6609 }
6610 else {
6611 emit_int8((unsigned char)0xC1);
6612 emit_operand(as_Register(7), src, 1);
6613 emit_int8(imm8);
6614 }
6615 }
6616
6617 void Assembler::sarl(Address dst) {
6618 InstructionMark im(this);
6619 prefix(dst);
6620 emit_int8((unsigned char)0xD3);
6621 emit_operand(as_Register(7), dst, 0);
6622 }
6623
6624 void Assembler::esarl(Register dst, Address src, bool no_flags) {
6625 InstructionMark im(this);
6626 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6627 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6628 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6629 emit_int8((unsigned char)0xD3);
6630 emit_operand(as_Register(7), src, 0);
6631 }
6632
6633 void Assembler::sarl(Register dst, int imm8) {
6634 int encode = prefix_and_encode(dst->encoding());
6635 assert(isShiftCount(imm8), "illegal shift count");
6636 if (imm8 == 1) {
6637 emit_int16((unsigned char)0xD1, (0xF8 | encode));
6638 } else {
6639 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
6640 }
6641 }
6642
6643 void Assembler::esarl(Register dst, Register src, int imm8, bool no_flags) {
6644 assert(isShiftCount(imm8), "illegal shift count");
6645 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6646 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6647 if (imm8 == 1) {
6648 emit_int16((unsigned char)0xD1, (0xF8 | encode));
6649 } else {
6650 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
6651 }
6652 }
6653
6654 void Assembler::sarl(Register dst) {
6655 int encode = prefix_and_encode(dst->encoding());
6656 emit_int16((unsigned char)0xD3, (0xF8 | encode));
6657 }
6658
6659 void Assembler::esarl(Register dst, Register src, bool no_flags) {
6660 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6661 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6662 emit_int16((unsigned char)0xD3, (0xF8 | encode));
6663 }
6664
6665 void Assembler::sbbl(Address dst, int32_t imm32) {
6666 InstructionMark im(this);
6667 prefix(dst);
6668 emit_arith_operand(0x81, rbx, dst, imm32);
6669 }
6670
6671 void Assembler::sbbl(Register dst, int32_t imm32) {
6672 prefix(dst);
6673 emit_arith(0x81, 0xD8, dst, imm32);
6674 }
6675
6676 void Assembler::sbbl(Register dst, Address src) {
6677 InstructionMark im(this);
6678 prefix(src, dst);
6679 emit_int8(0x1B);
6680 emit_operand(dst, src, 0);
6681 }
6682
6683 void Assembler::sbbl(Register dst, Register src) {
6684 (void) prefix_and_encode(dst->encoding(), src->encoding());
6685 emit_arith(0x1B, 0xC0, dst, src);
6686 }
6687
6688 void Assembler::setb(Condition cc, Register dst) {
6689 assert(0 <= cc && cc < 16, "illegal cc");
6690 int encode = prefix_and_encode(dst->encoding(), true, true /* is_map1 */);
6691 emit_opcode_prefix_and_encoding((unsigned char)0x90 | cc, 0xC0, encode);
6692 }
6693
6694 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
6695 assert(VM_Version::supports_ssse3(), "");
6696 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6697 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6698 emit_int24(0x0F, (0xC0 | encode), imm8);
6699 }
6700
6701 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
6702 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
6703 vector_len == AVX_256bit? VM_Version::supports_avx2() :
6704 0, "");
6705 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6706 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6707 emit_int24(0x0F, (0xC0 | encode), imm8);
6708 }
6709
6710 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6711 assert(VM_Version::supports_evex(), "");
6712 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6713 attributes.set_is_evex_instruction();
6714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6715 emit_int24(0x3, (0xC0 | encode), imm8);
6716 }
6717
6718 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
6719 assert(VM_Version::supports_sse4_1(), "");
6720 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6721 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6722 emit_int24(0x0E, (0xC0 | encode), imm8);
6723 }
6724
6725 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
6726 assert(VM_Version::supports_sha(), "");
6727 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
6728 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
6729 }
6730
6731 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
6732 assert(VM_Version::supports_sha(), "");
6733 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6734 emit_int16((unsigned char)0xC8, (0xC0 | encode));
6735 }
6736
6737 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
6738 assert(VM_Version::supports_sha(), "");
6739 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6740 emit_int16((unsigned char)0xC9, (0xC0 | encode));
6741 }
6742
6743 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
6744 assert(VM_Version::supports_sha(), "");
6745 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6746 emit_int16((unsigned char)0xCA, (0xC0 | encode));
6747 }
6748
6749 // xmm0 is implicit additional source to this instruction.
6750 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
6751 assert(VM_Version::supports_sha(), "");
6752 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6753 emit_int16((unsigned char)0xCB, (0xC0 | encode));
6754 }
6755
6756 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
6757 assert(VM_Version::supports_sha(), "");
6758 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6759 emit_int16((unsigned char)0xCC, (0xC0 | encode));
6760 }
6761
6762 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
6763 assert(VM_Version::supports_sha(), "");
6764 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
6765 emit_int16((unsigned char)0xCD, (0xC0 | encode));
6766 }
6767
6768 void Assembler::sha512msg1(XMMRegister dst, XMMRegister src) {
6769 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
6770 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6771 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
6772 emit_int16((unsigned char)0xCC, (0xC0 | encode));
6773 }
6774
6775 void Assembler::sha512msg2(XMMRegister dst, XMMRegister src) {
6776 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
6777 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6778 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
6779 emit_int16((unsigned char)0xCD, (0xC0 | encode));
6780 }
6781
6782 void Assembler::sha512rnds2(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6783 assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
6784 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6785 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
6786 emit_int16((unsigned char)0xCB, (0xC0 | encode));
6787 }
6788
6789 void Assembler::shll(Register dst, int imm8) {
6790 assert(isShiftCount(imm8), "illegal shift count");
6791 int encode = prefix_and_encode(dst->encoding());
6792 if (imm8 == 1 ) {
6793 emit_int16((unsigned char)0xD1, (0xE0 | encode));
6794 } else {
6795 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
6796 }
6797 }
6798
6799 void Assembler::eshll(Register dst, Register src, int imm8, bool no_flags) {
6800 assert(isShiftCount(imm8), "illegal shift count");
6801 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6802 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6803 if (imm8 == 1 ) {
6804 emit_int16((unsigned char)0xD1, (0xE0 | encode));
6805 } else {
6806 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
6807 }
6808 }
6809
6810 void Assembler::shll(Register dst) {
6811 int encode = prefix_and_encode(dst->encoding());
6812 emit_int16((unsigned char)0xD3, (0xE0 | encode));
6813 }
6814
6815 void Assembler::eshll(Register dst, Register src, bool no_flags) {
6816 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6817 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6818 emit_int16((unsigned char)0xD3, (0xE0 | encode));
6819 }
6820
6821 void Assembler::shrl(Register dst, int imm8) {
6822 assert(isShiftCount(imm8), "illegal shift count");
6823 int encode = prefix_and_encode(dst->encoding());
6824 if (imm8 == 1) {
6825 emit_int16((unsigned char)0xD1, (0xE8 | encode));
6826 }
6827 else {
6828 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
6829 }
6830 }
6831
6832 void Assembler::eshrl(Register dst, Register src, int imm8, bool no_flags) {
6833 assert(isShiftCount(imm8), "illegal shift count");
6834 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6835 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6836 if (imm8 == 1) {
6837 emit_int16((unsigned char)0xD1, (0xE8 | encode));
6838 }
6839 else {
6840 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
6841 }
6842 }
6843
6844 void Assembler::shrl(Register dst) {
6845 int encode = prefix_and_encode(dst->encoding());
6846 emit_int16((unsigned char)0xD3, (0xE8 | encode));
6847 }
6848
6849 void Assembler::eshrl(Register dst, Register src, bool no_flags) {
6850 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6851 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6852 emit_int16((unsigned char)0xD3, (0xE8 | encode));
6853 }
6854
6855 void Assembler::shrl(Address dst) {
6856 InstructionMark im(this);
6857 prefix(dst);
6858 emit_int8((unsigned char)0xD3);
6859 emit_operand(as_Register(5), dst, 0);
6860 }
6861
6862 void Assembler::eshrl(Register dst, Address src, bool no_flags) {
6863 InstructionMark im(this);
6864 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6865 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6866 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6867 emit_int8((unsigned char)0xD3);
6868 emit_operand(as_Register(5), src, 0);
6869 }
6870
6871 void Assembler::shrl(Address dst, int imm8) {
6872 InstructionMark im(this);
6873 assert(isShiftCount(imm8), "illegal shift count");
6874 prefix(dst);
6875 if (imm8 == 1) {
6876 emit_int8((unsigned char)0xD1);
6877 emit_operand(as_Register(5), dst, 0);
6878 }
6879 else {
6880 emit_int8((unsigned char)0xC1);
6881 emit_operand(as_Register(5), dst, 1);
6882 emit_int8(imm8);
6883 }
6884 }
6885
6886 void Assembler::eshrl(Register dst, Address src, int imm8, bool no_flags) {
6887 InstructionMark im(this);
6888 assert(isShiftCount(imm8), "illegal shift count");
6889 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6890 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
6891 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
6892 if (imm8 == 1) {
6893 emit_int8((unsigned char)0xD1);
6894 emit_operand(as_Register(5), src, 0);
6895 }
6896 else {
6897 emit_int8((unsigned char)0xC1);
6898 emit_operand(as_Register(5), src, 1);
6899 emit_int8(imm8);
6900 }
6901 }
6902
6903 void Assembler::shldl(Register dst, Register src) {
6904 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6905 emit_opcode_prefix_and_encoding((unsigned char)0xA5, 0xC0, encode);
6906 }
6907
6908 void Assembler::eshldl(Register dst, Register src1, Register src2, bool no_flags) {
6909 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xA5, no_flags, true /* is_map1 */);
6910 }
6911
6912 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6913 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6914 emit_opcode_prefix_and_encoding((unsigned char)0xA4, 0xC0, encode, imm8);
6915 }
6916
6917 void Assembler::eshldl(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) {
6918 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x24, no_flags, true /* is_map1 */);
6919 }
6920
6921 void Assembler::shrdl(Register dst, Register src) {
6922 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6923 emit_opcode_prefix_and_encoding((unsigned char)0xAD, 0xC0, encode);
6924 }
6925
6926 void Assembler::eshrdl(Register dst, Register src1, Register src2, bool no_flags) {
6927 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAD, no_flags, true /* is_map1 */);
6928 }
6929
6930 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
6931 int encode = prefix_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6932 emit_opcode_prefix_and_encoding((unsigned char)0xAC, 0xC0, encode, imm8);
6933 }
6934
6935 void Assembler::eshrdl(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) {
6936 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x2C, no_flags, true /* is_map1 */);
6937 }
6938
6939 void Assembler::shldq(Register dst, Register src, int8_t imm8) {
6940 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6941 emit_opcode_prefix_and_encoding((unsigned char)0xA4, 0xC0, encode, imm8);
6942 }
6943
6944 void Assembler::eshldq(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) {
6945 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x24, no_flags, true /* is_map1 */);
6946 }
6947
6948 void Assembler::shrdq(Register dst, Register src, int8_t imm8) {
6949 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
6950 emit_opcode_prefix_and_encoding((unsigned char)0xAC, 0xC0, encode, imm8);
6951 }
6952
6953 void Assembler::eshrdq(Register dst, Register src1, Register src2, int8_t imm8, bool no_flags) {
6954 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), imm8, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x2C, no_flags, true /* is_map1 */);
6955 }
6956
6957 // copies a single word from [esi] to [edi]
6958 void Assembler::smovl() {
6959 emit_int8((unsigned char)0xA5);
6960 }
6961
6962 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
6963 assert(VM_Version::supports_sse4_1(), "");
6964 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6965 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6966 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
6967 }
6968
6969 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
6970 assert(VM_Version::supports_sse4_1(), "");
6971 InstructionMark im(this);
6972 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6973 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6974 emit_int8(0x0B);
6975 emit_operand(dst, src, 1);
6976 emit_int8((unsigned char)rmode);
6977 }
6978
6979 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
6980 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6981 attributes.set_rex_vex_w_reverted();
6982 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6983 emit_int16(0x51, (0xC0 | encode));
6984 }
6985
6986 void Assembler::sqrtsd(XMMRegister dst, Address src) {
6987 InstructionMark im(this);
6988 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6989 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6990 attributes.set_rex_vex_w_reverted();
6991 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6992 emit_int8(0x51);
6993 emit_operand(dst, src, 0);
6994 }
6995
6996 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
6997 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6998 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6999 emit_int16(0x51, (0xC0 | encode));
7000 }
7001
7002 void Assembler::std() {
7003 emit_int8((unsigned char)0xFD);
7004 }
7005
7006 void Assembler::sqrtss(XMMRegister dst, Address src) {
7007 InstructionMark im(this);
7008 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7010 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7011 emit_int8(0x51);
7012 emit_operand(dst, src, 0);
7013 }
7014
7015 void Assembler::stmxcsr(Address dst) {
7016 // This instruction should be SSE encoded with the REX2 prefix when an
7017 // extended GPR is present. To be consistent when UseAPX is enabled, use
7018 // this encoding even when an extended GPR is not used.
7019 if (UseAVX > 0 && !UseAPX ) {
7020 assert(VM_Version::supports_avx(), "");
7021 InstructionMark im(this);
7022 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7023 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7024 emit_int8((unsigned char)0xAE);
7025 emit_operand(as_Register(3), dst, 0);
7026 } else {
7027 InstructionMark im(this);
7028 prefix(dst, true /* is_map1 */);
7029 emit_int8((unsigned char)0xAE);
7030 emit_operand(as_Register(3), dst, 0);
7031 }
7032 }
7033
7034 void Assembler::subl(Address dst, int32_t imm32) {
7035 InstructionMark im(this);
7036 prefix(dst);
7037 emit_arith_operand(0x81, rbp, dst, imm32);
7038 }
7039
7040 void Assembler::esubl(Register dst, Address src, int32_t imm32, bool no_flags) {
7041 InstructionMark im(this);
7042 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7043 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
7044 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7045 emit_arith_operand(0x81, rbp, src, imm32);
7046 }
7047
7048 void Assembler::subl(Address dst, Register src) {
7049 InstructionMark im(this);
7050 prefix(dst, src);
7051 emit_int8(0x29);
7052 emit_operand(src, dst, 0);
7053 }
7054
7055 void Assembler::esubl(Register dst, Address src1, Register src2, bool no_flags) {
7056 InstructionMark im(this);
7057 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7058 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
7059 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7060 emit_int8(0x29);
7061 emit_operand(src2, src1, 0);
7062 }
7063
7064 void Assembler::subl(Register dst, int32_t imm32) {
7065 prefix(dst);
7066 emit_arith(0x81, 0xE8, dst, imm32);
7067 }
7068
7069 void Assembler::esubl(Register dst, Register src, int32_t imm32, bool no_flags) {
7070 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xE8, no_flags);
7071 }
7072
7073 // Force generation of a 4 byte immediate value even if it fits into 8bit
7074 void Assembler::subl_imm32(Register dst, int32_t imm32) {
7075 prefix(dst);
7076 emit_arith_imm32(0x81, 0xE8, dst, imm32);
7077 }
7078
7079 void Assembler::esubl_imm32(Register dst, Register src, int32_t imm32, bool no_flags) {
7080 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7081 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7082 emit_arith_imm32(0x81, 0xE8, src, imm32);
7083 }
7084
7085 void Assembler::subl(Register dst, Address src) {
7086 InstructionMark im(this);
7087 prefix(src, dst);
7088 emit_int8(0x2B);
7089 emit_operand(dst, src, 0);
7090 }
7091
7092 void Assembler::esubl(Register dst, Register src1, Address src2, bool no_flags) {
7093 InstructionMark im(this);
7094 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x2B, no_flags);
7095 }
7096
7097 void Assembler::subl(Register dst, Register src) {
7098 (void) prefix_and_encode(dst->encoding(), src->encoding());
7099 emit_arith(0x2B, 0xC0, dst, src);
7100 }
7101
7102 void Assembler::esubl(Register dst, Register src1, Register src2, bool no_flags) {
7103 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7104 // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
7105 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
7106 (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7107 emit_arith(0x2B, 0xC0, src1, src2);
7108 }
7109
7110 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
7111 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7112 attributes.set_rex_vex_w_reverted();
7113 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7114 emit_int16(0x5C, (0xC0 | encode));
7115 }
7116
7117 void Assembler::subsd(XMMRegister dst, Address src) {
7118 InstructionMark im(this);
7119 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7120 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7121 attributes.set_rex_vex_w_reverted();
7122 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7123 emit_int8(0x5C);
7124 emit_operand(dst, src, 0);
7125 }
7126
7127 void Assembler::subss(XMMRegister dst, XMMRegister src) {
7128 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
7129 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7130 emit_int16(0x5C, (0xC0 | encode));
7131 }
7132
7133 void Assembler::subss(XMMRegister dst, Address src) {
7134 InstructionMark im(this);
7135 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7136 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7137 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7138 emit_int8(0x5C);
7139 emit_operand(dst, src, 0);
7140 }
7141
7142 void Assembler::testb(Register dst, int imm8, bool use_ral) {
7143 if (dst == rax) {
7144 if (use_ral) {
7145 emit_int8((unsigned char)0xA8);
7146 emit_int8(imm8);
7147 } else {
7148 emit_int8((unsigned char)0xF6);
7149 emit_int8((unsigned char)0xC4);
7150 emit_int8(imm8);
7151 }
7152 } else {
7153 (void) prefix_and_encode(dst->encoding(), true);
7154 emit_arith_b(0xF6, 0xC0, dst, imm8);
7155 }
7156 }
7157
7158 void Assembler::testb(Address dst, int imm8) {
7159 InstructionMark im(this);
7160 prefix(dst);
7161 emit_int8((unsigned char)0xF6);
7162 emit_operand(rax, dst, 1);
7163 emit_int8(imm8);
7164 }
7165
7166 void Assembler::testl(Address dst, int32_t imm32) {
7167 InstructionMark im(this);
7168 prefix(dst);
7169 emit_int8((unsigned char)0xF7);
7170 emit_operand(as_Register(0), dst, 4);
7171 emit_int32(imm32);
7172 }
7173
7174 void Assembler::testl(Register dst, int32_t imm32) {
7175 // not using emit_arith because test
7176 // doesn't support sign-extension of
7177 // 8bit operands
7178 if (dst == rax) {
7179 emit_int8((unsigned char)0xA9);
7180 emit_int32(imm32);
7181 } else {
7182 int encode = dst->encoding();
7183 encode = prefix_and_encode(encode);
7184 emit_int16((unsigned char)0xF7, (0xC0 | encode));
7185 emit_int32(imm32);
7186 }
7187 }
7188
7189 void Assembler::testl(Register dst, Register src) {
7190 (void) prefix_and_encode(dst->encoding(), src->encoding());
7191 emit_arith(0x85, 0xC0, dst, src);
7192 }
7193
7194 void Assembler::testl(Register dst, Address src) {
7195 InstructionMark im(this);
7196 prefix(src, dst);
7197 emit_int8((unsigned char)0x85);
7198 emit_operand(dst, src, 0);
7199 }
7200
7201 void Assembler::tzcntl(Register dst, Register src) {
7202 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7203 emit_int8((unsigned char)0xF3);
7204 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
7205 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
7206 }
7207
7208 void Assembler::etzcntl(Register dst, Register src, bool no_flags) {
7209 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7210 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7211 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7212 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7213 }
7214
7215 void Assembler::tzcntl(Register dst, Address src) {
7216 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7217 InstructionMark im(this);
7218 emit_int8((unsigned char)0xF3);
7219 prefix(src, dst, false, true /* is_map1 */);
7220 emit_int8((unsigned char)0xBC);
7221 emit_operand(dst, src, 0);
7222 }
7223
7224 void Assembler::etzcntl(Register dst, Address src, bool no_flags) {
7225 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7226 InstructionMark im(this);
7227 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7228 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
7229 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7230 emit_int8((unsigned char)0xF4);
7231 emit_operand(dst, src, 0);
7232 }
7233
7234 void Assembler::tzcntq(Register dst, Register src) {
7235 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7236 emit_int8((unsigned char)0xF3);
7237 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
7238 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
7239 }
7240
7241 void Assembler::etzcntq(Register dst, Register src, bool no_flags) {
7242 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7243 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7244 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7245 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7246 }
7247
7248 void Assembler::tzcntq(Register dst, Address src) {
7249 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7250 InstructionMark im(this);
7251 emit_int8((unsigned char)0xF3);
7252 prefixq(src, dst, true /* is_map1 */);
7253 emit_int8((unsigned char)0xBC);
7254 emit_operand(dst, src, 0);
7255 }
7256
7257 void Assembler::etzcntq(Register dst, Address src, bool no_flags) {
7258 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
7259 InstructionMark im(this);
7260 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7261 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
7262 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7263 emit_int8((unsigned char)0xF4);
7264 emit_operand(dst, src, 0);
7265 }
7266
7267 void Assembler::ucomisd(XMMRegister dst, Address src) {
7268 InstructionMark im(this);
7269 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7270 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7271 attributes.set_rex_vex_w_reverted();
7272 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7273 emit_int8(0x2E);
7274 emit_operand(dst, src, 0);
7275 }
7276
7277 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
7278 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7279 attributes.set_rex_vex_w_reverted();
7280 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7281 emit_int16(0x2E, (0xC0 | encode));
7282 }
7283
7284 void Assembler::ucomiss(XMMRegister dst, Address src) {
7285 InstructionMark im(this);
7286 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7287 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7288 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7289 emit_int8(0x2E);
7290 emit_operand(dst, src, 0);
7291 }
7292
7293 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
7294 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7295 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7296 emit_int16(0x2E, (0xC0 | encode));
7297 }
7298
7299 void Assembler::xabort(int8_t imm8) {
7300 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
7301 }
7302
7303 void Assembler::xaddb(Address dst, Register src) {
7304 InstructionMark im(this);
7305 prefix(dst, src, true, true /* is_map1 */);
7306 emit_int8((unsigned char)0xC0);
7307 emit_operand(src, dst, 0);
7308 }
7309
7310 void Assembler::xaddw(Address dst, Register src) {
7311 InstructionMark im(this);
7312 emit_int8(0x66);
7313 prefix(dst, src, false, true /* is_map1 */);
7314 emit_int8((unsigned char)0xC1);
7315 emit_operand(src, dst, 0);
7316 }
7317
7318 void Assembler::xaddl(Address dst, Register src) {
7319 InstructionMark im(this);
7320 prefix(dst, src, false, true /* is_map1 */);
7321 emit_int8((unsigned char)0xC1);
7322 emit_operand(src, dst, 0);
7323 }
7324
7325 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
7326 InstructionMark im(this);
7327 relocate(rtype);
7328 if (abort.is_bound()) {
7329 address entry = target(abort);
7330 assert(entry != nullptr, "abort entry null");
7331 int offset = checked_cast<int>(entry - pc());
7332 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
7333 emit_int32(offset - 6); // 2 opcode + 4 address
7334 } else {
7335 abort.add_patch_at(code(), locator());
7336 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
7337 emit_int32(0);
7338 }
7339 }
7340
7341 void Assembler::xchgb(Register dst, Address src) { // xchg
7342 InstructionMark im(this);
7343 prefix(src, dst, true);
7344 emit_int8((unsigned char)0x86);
7345 emit_operand(dst, src, 0);
7346 }
7347
7348 void Assembler::xchgw(Register dst, Address src) { // xchg
7349 InstructionMark im(this);
7350 emit_int8(0x66);
7351 prefix(src, dst);
7352 emit_int8((unsigned char)0x87);
7353 emit_operand(dst, src, 0);
7354 }
7355
7356 void Assembler::xchgl(Register dst, Address src) { // xchg
7357 InstructionMark im(this);
7358 prefix(src, dst);
7359 emit_int8((unsigned char)0x87);
7360 emit_operand(dst, src, 0);
7361 }
7362
7363 void Assembler::xchgl(Register dst, Register src) {
7364 int encode = prefix_and_encode(dst->encoding(), src->encoding());
7365 emit_int16((unsigned char)0x87, (0xC0 | encode));
7366 }
7367
7368 void Assembler::xend() {
7369 emit_int24(0x0F, 0x01, (unsigned char)0xD5);
7370 }
7371
7372 void Assembler::xgetbv() {
7373 emit_int24(0x0F, 0x01, (unsigned char)0xD0);
7374 }
7375
7376 void Assembler::xorl(Address dst, int32_t imm32) {
7377 InstructionMark im(this);
7378 prefix(dst);
7379 emit_arith_operand(0x81, as_Register(6), dst, imm32);
7380 }
7381
7382 void Assembler::exorl(Register dst, Address src, int32_t imm32, bool no_flags) {
7383 InstructionMark im(this);
7384 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7385 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
7386 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
7387 emit_arith_operand(0x81, as_Register(6), src, imm32);
7388 }
7389
7390 void Assembler::xorl(Register dst, int32_t imm32) {
7391 prefix(dst);
7392 emit_arith(0x81, 0xF0, dst, imm32);
7393 }
7394
7395 void Assembler::exorl(Register dst, Register src, int32_t imm32, bool no_flags) {
7396 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x81, 0xF0, no_flags);
7397 }
7398
7399 void Assembler::xorl(Register dst, Address src) {
7400 InstructionMark im(this);
7401 prefix(src, dst);
7402 emit_int8(0x33);
7403 emit_operand(dst, src, 0);
7404 }
7405
7406 void Assembler::exorl(Register dst, Register src1, Address src2, bool no_flags) {
7407 InstructionMark im(this);
7408 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, no_flags);
7409 }
7410
7411 void Assembler::xorl(Register dst, Register src) {
7412 (void) prefix_and_encode(dst->encoding(), src->encoding());
7413 emit_arith(0x33, 0xC0, dst, src);
7414 }
7415
7416 void Assembler::exorl(Register dst, Register src1, Register src2, bool no_flags) {
7417 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
7418 }
7419
7420 void Assembler::xorl(Address dst, Register src) {
7421 InstructionMark im(this);
7422 prefix(dst, src);
7423 emit_int8(0x31);
7424 emit_operand(src, dst, 0);
7425 }
7426
7427 void Assembler::exorl(Register dst, Address src1, Register src2, bool no_flags) {
7428 InstructionMark im(this);
7429 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
7430 }
7431
7432 void Assembler::xorb(Register dst, Address src) {
7433 InstructionMark im(this);
7434 prefix(src, dst);
7435 emit_int8(0x32);
7436 emit_operand(dst, src, 0);
7437 }
7438
7439 void Assembler::exorb(Register dst, Register src1, Address src2, bool no_flags) {
7440 InstructionMark im(this);
7441 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x32, no_flags);
7442 }
7443
7444 void Assembler::xorb(Address dst, Register src) {
7445 InstructionMark im(this);
7446 prefix(dst, src, true);
7447 emit_int8(0x30);
7448 emit_operand(src, dst, 0);
7449 }
7450
7451 void Assembler::exorb(Register dst, Address src1, Register src2, bool no_flags) {
7452 InstructionMark im(this);
7453 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x30, no_flags, false /* is_map1 */, true /* is_commutative */);
7454 }
7455
7456 void Assembler::xorw(Register dst, Address src) {
7457 InstructionMark im(this);
7458 emit_int8(0x66);
7459 prefix(src, dst);
7460 emit_int8(0x33);
7461 emit_operand(dst, src, 0);
7462 }
7463
7464 void Assembler::exorw(Register dst, Register src1, Address src2, bool no_flags) {
7465 InstructionMark im(this);
7466 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x33, no_flags);
7467 }
7468
7469 // AVX 3-operands scalar float-point arithmetic instructions
7470
7471 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
7472 assert(VM_Version::supports_avx(), "");
7473 InstructionMark im(this);
7474 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7475 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7476 attributes.set_rex_vex_w_reverted();
7477 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7478 emit_int8(0x58);
7479 emit_operand(dst, src, 0);
7480 }
7481
7482 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7483 assert(VM_Version::supports_avx(), "");
7484 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7485 attributes.set_rex_vex_w_reverted();
7486 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7487 emit_int16(0x58, (0xC0 | encode));
7488 }
7489
7490 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
7491 assert(VM_Version::supports_avx(), "");
7492 InstructionMark im(this);
7493 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7494 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7495 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7496 emit_int8(0x58);
7497 emit_operand(dst, src, 0);
7498 }
7499
7500 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7501 assert(VM_Version::supports_avx(), "");
7502 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7503 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7504 emit_int16(0x58, (0xC0 | encode));
7505 }
7506
7507 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
7508 assert(VM_Version::supports_avx(), "");
7509 InstructionMark im(this);
7510 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7511 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7512 attributes.set_rex_vex_w_reverted();
7513 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7514 emit_int8(0x5E);
7515 emit_operand(dst, src, 0);
7516 }
7517
7518 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7519 assert(VM_Version::supports_avx(), "");
7520 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7521 attributes.set_rex_vex_w_reverted();
7522 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7523 emit_int16(0x5E, (0xC0 | encode));
7524 }
7525
7526 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
7527 assert(VM_Version::supports_avx(), "");
7528 InstructionMark im(this);
7529 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7530 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7531 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7532 emit_int8(0x5E);
7533 emit_operand(dst, src, 0);
7534 }
7535
7536 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7537 assert(VM_Version::supports_avx(), "");
7538 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7539 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7540 emit_int16(0x5E, (0xC0 | encode));
7541 }
7542
7543 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
7544 assert(VM_Version::supports_fma(), "");
7545 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7546 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7547 emit_int16((unsigned char)0xB9, (0xC0 | encode));
7548 }
7549
7550 void Assembler::evfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2, EvexRoundPrefix rmode) { // Need to add rmode for rounding mode support
7551 assert(VM_Version::supports_evex(), "");
7552 InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7553 attributes.set_extended_context();
7554 attributes.set_is_evex_instruction();
7555 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7556 emit_int16((unsigned char)0xAD, (0xC0 | encode));
7557 }
7558
7559 void Assembler::vfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
7560 assert(VM_Version::supports_fma(), "");
7561 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7562 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7563 emit_int16((unsigned char)0xAD, (0xC0 | encode));
7564 }
7565
7566 void Assembler::vfnmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
7567 assert(VM_Version::supports_fma(), "");
7568 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7569 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7570 emit_int16((unsigned char)0xBD, (0xC0 | encode));
7571 }
7572
7573 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
7574 assert(VM_Version::supports_fma(), "");
7575 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7576 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7577 emit_int16((unsigned char)0xB9, (0xC0 | encode));
7578 }
7579
7580 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
7581 assert(VM_Version::supports_avx(), "");
7582 InstructionMark im(this);
7583 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7584 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7585 attributes.set_rex_vex_w_reverted();
7586 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7587 emit_int8(0x59);
7588 emit_operand(dst, src, 0);
7589 }
7590
7591 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7592 assert(VM_Version::supports_avx(), "");
7593 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7594 attributes.set_rex_vex_w_reverted();
7595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7596 emit_int16(0x59, (0xC0 | encode));
7597 }
7598
7599 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
7600 assert(VM_Version::supports_avx(), "");
7601 InstructionMark im(this);
7602 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7603 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7604 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7605 emit_int8(0x59);
7606 emit_operand(dst, src, 0);
7607 }
7608
7609 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7610 assert(VM_Version::supports_avx(), "");
7611 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7612 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7613 emit_int16(0x59, (0xC0 | encode));
7614 }
7615
7616 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
7617 assert(VM_Version::supports_avx(), "");
7618 InstructionMark im(this);
7619 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7620 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7621 attributes.set_rex_vex_w_reverted();
7622 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7623 emit_int8(0x5C);
7624 emit_operand(dst, src, 0);
7625 }
7626
7627 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7628 assert(VM_Version::supports_avx(), "");
7629 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7630 attributes.set_rex_vex_w_reverted();
7631 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7632 emit_int16(0x5C, (0xC0 | encode));
7633 }
7634
7635 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
7636 assert(VM_Version::supports_avx(), "");
7637 InstructionMark im(this);
7638 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7639 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
7640 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7641 emit_int8(0x5C);
7642 emit_operand(dst, src, 0);
7643 }
7644
7645 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7646 assert(VM_Version::supports_avx(), "");
7647 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7648 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7649 emit_int16(0x5C, (0xC0 | encode));
7650 }
7651
7652 //====================VECTOR ARITHMETIC=====================================
7653
7654 // Float-point vector arithmetic
7655
7656 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
7657 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7658 attributes.set_rex_vex_w_reverted();
7659 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7660 emit_int16(0x58, (0xC0 | encode));
7661 }
7662
7663 void Assembler::addpd(XMMRegister dst, Address src) {
7664 InstructionMark im(this);
7665 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7666 attributes.set_rex_vex_w_reverted();
7667 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7668 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7669 emit_int8(0x58);
7670 emit_operand(dst, src, 0);
7671 }
7672
7673
7674 void Assembler::addps(XMMRegister dst, XMMRegister src) {
7675 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7676 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7677 emit_int16(0x58, (0xC0 | encode));
7678 }
7679
7680 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7681 assert(VM_Version::supports_avx(), "");
7682 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7683 attributes.set_rex_vex_w_reverted();
7684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7685 emit_int16(0x58, (0xC0 | encode));
7686 }
7687
7688 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7689 assert(VM_Version::supports_avx(), "");
7690 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7691 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7692 emit_int16(0x58, (0xC0 | encode));
7693 }
7694
7695 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7696 assert(VM_Version::supports_avx(), "");
7697 InstructionMark im(this);
7698 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7699 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7700 attributes.set_rex_vex_w_reverted();
7701 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7702 emit_int8(0x58);
7703 emit_operand(dst, src, 0);
7704 }
7705
7706 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7707 assert(VM_Version::supports_avx(), "");
7708 InstructionMark im(this);
7709 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7710 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7711 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7712 emit_int8(0x58);
7713 emit_operand(dst, src, 0);
7714 }
7715
7716 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
7717 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7718 attributes.set_rex_vex_w_reverted();
7719 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7720 emit_int16(0x5C, (0xC0 | encode));
7721 }
7722
7723 void Assembler::subps(XMMRegister dst, XMMRegister src) {
7724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7725 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7726 emit_int16(0x5C, (0xC0 | encode));
7727 }
7728
7729 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7730 assert(VM_Version::supports_avx(), "");
7731 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7732 attributes.set_rex_vex_w_reverted();
7733 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7734 emit_int16(0x5C, (0xC0 | encode));
7735 }
7736
7737 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7738 assert(VM_Version::supports_avx(), "");
7739 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7740 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7741 emit_int16(0x5C, (0xC0 | encode));
7742 }
7743
7744 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7745 assert(VM_Version::supports_avx(), "");
7746 InstructionMark im(this);
7747 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7748 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7749 attributes.set_rex_vex_w_reverted();
7750 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7751 emit_int8(0x5C);
7752 emit_operand(dst, src, 0);
7753 }
7754
7755 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7756 assert(VM_Version::supports_avx(), "");
7757 InstructionMark im(this);
7758 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7759 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7760 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7761 emit_int8(0x5C);
7762 emit_operand(dst, src, 0);
7763 }
7764
7765 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
7766 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7767 attributes.set_rex_vex_w_reverted();
7768 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7769 emit_int16(0x59, (0xC0 | encode));
7770 }
7771
7772 void Assembler::mulpd(XMMRegister dst, Address src) {
7773 InstructionMark im(this);
7774 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7775 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7776 attributes.set_rex_vex_w_reverted();
7777 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7778 emit_int8(0x59);
7779 emit_operand(dst, src, 0);
7780 }
7781
7782 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
7783 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7784 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7785 emit_int16(0x59, (0xC0 | encode));
7786 }
7787
7788 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7789 assert(VM_Version::supports_avx(), "");
7790 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7791 attributes.set_rex_vex_w_reverted();
7792 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7793 emit_int16(0x59, (0xC0 | encode));
7794 }
7795
7796 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7797 assert(VM_Version::supports_avx(), "");
7798 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7799 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7800 emit_int16(0x59, (0xC0 | encode));
7801 }
7802
7803 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7804 assert(VM_Version::supports_avx(), "");
7805 InstructionMark im(this);
7806 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7807 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7808 attributes.set_rex_vex_w_reverted();
7809 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7810 emit_int8(0x59);
7811 emit_operand(dst, src, 0);
7812 }
7813
7814 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7815 assert(VM_Version::supports_avx(), "");
7816 InstructionMark im(this);
7817 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7818 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7819 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7820 emit_int8(0x59);
7821 emit_operand(dst, src, 0);
7822 }
7823
7824 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
7825 assert(VM_Version::supports_fma(), "");
7826 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7827 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7828 emit_int16((unsigned char)0xB8, (0xC0 | encode));
7829 }
7830
7831 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
7832 assert(VM_Version::supports_fma(), "");
7833 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7834 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7835 emit_int16((unsigned char)0xB8, (0xC0 | encode));
7836 }
7837
7838 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
7839 assert(VM_Version::supports_fma(), "");
7840 InstructionMark im(this);
7841 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7842 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7843 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7844 emit_int8((unsigned char)0xB8);
7845 emit_operand(dst, src2, 0);
7846 }
7847
7848 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
7849 assert(VM_Version::supports_fma(), "");
7850 InstructionMark im(this);
7851 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7852 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7853 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7854 emit_int8((unsigned char)0xB8);
7855 emit_operand(dst, src2, 0);
7856 }
7857
7858 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
7859 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7860 attributes.set_rex_vex_w_reverted();
7861 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7862 emit_int16(0x5E, (0xC0 | encode));
7863 }
7864
7865 void Assembler::divps(XMMRegister dst, XMMRegister src) {
7866 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7867 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7868 emit_int16(0x5E, (0xC0 | encode));
7869 }
7870
7871 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7872 assert(VM_Version::supports_avx(), "");
7873 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7874 attributes.set_rex_vex_w_reverted();
7875 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7876 emit_int16(0x5E, (0xC0 | encode));
7877 }
7878
7879 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7880 assert(VM_Version::supports_avx(), "");
7881 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7882 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7883 emit_int16(0x5E, (0xC0 | encode));
7884 }
7885
7886 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7887 assert(VM_Version::supports_avx(), "");
7888 InstructionMark im(this);
7889 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7890 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7891 attributes.set_rex_vex_w_reverted();
7892 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7893 emit_int8(0x5E);
7894 emit_operand(dst, src, 0);
7895 }
7896
7897 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7898 assert(VM_Version::supports_avx(), "");
7899 InstructionMark im(this);
7900 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7901 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7902 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7903 emit_int8(0x5E);
7904 emit_operand(dst, src, 0);
7905 }
7906
7907 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
7908 assert(VM_Version::supports_avx(), "");
7909 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7911 emit_int24(0x09, (0xC0 | encode), (rmode));
7912 }
7913
7914 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
7915 assert(VM_Version::supports_avx(), "");
7916 assert(!needs_eevex(src.base(), src.index()), "does not support extended gprs as BASE or INDEX of address operand");
7917 InstructionMark im(this);
7918 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7919 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7920 emit_int8(0x09);
7921 emit_operand(dst, src, 1);
7922 emit_int8((rmode));
7923 }
7924
7925 void Assembler::vroundsd(XMMRegister dst, XMMRegister src, XMMRegister src2, int32_t rmode) {
7926 assert(VM_Version::supports_avx(), "");
7927 assert(rmode <= 0x0f, "rmode 0x%x", rmode);
7928 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7929 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7930 emit_int24(0x0B, (0xC0 | encode), (rmode));
7931 }
7932
7933 void Assembler::vrndscalesd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int32_t rmode) {
7934 assert(VM_Version::supports_evex(), "requires EVEX support");
7935 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7936 attributes.set_is_evex_instruction();
7937 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7938 emit_int24(0x0B, (0xC0 | encode), (rmode));
7939 }
7940
7941 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
7942 assert(VM_Version::supports_evex(), "requires EVEX support");
7943 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7944 attributes.set_is_evex_instruction();
7945 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7946 emit_int24(0x09, (0xC0 | encode), (rmode));
7947 }
7948
7949 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
7950 assert(VM_Version::supports_evex(), "requires EVEX support");
7951 assert(dst != xnoreg, "sanity");
7952 InstructionMark im(this);
7953 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7954 attributes.set_is_evex_instruction();
7955 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7956 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7957 emit_int8(0x09);
7958 emit_operand(dst, src, 1);
7959 emit_int8((rmode));
7960 }
7961
7962 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
7963 assert(VM_Version::supports_avx(), "");
7964 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7965 attributes.set_rex_vex_w_reverted();
7966 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7967 emit_int16(0x51, (0xC0 | encode));
7968 }
7969
7970 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
7971 assert(VM_Version::supports_avx(), "");
7972 InstructionMark im(this);
7973 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7974 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7975 attributes.set_rex_vex_w_reverted();
7976 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7977 emit_int8(0x51);
7978 emit_operand(dst, src, 0);
7979 }
7980
7981 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
7982 assert(VM_Version::supports_avx(), "");
7983 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7984 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7985 emit_int16(0x51, (0xC0 | encode));
7986 }
7987
7988 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
7989 assert(VM_Version::supports_avx(), "");
7990 InstructionMark im(this);
7991 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7992 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7993 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7994 emit_int8(0x51);
7995 emit_operand(dst, src, 0);
7996 }
7997
7998 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
7999 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8000 attributes.set_rex_vex_w_reverted();
8001 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8002 emit_int16(0x54, (0xC0 | encode));
8003 }
8004
8005 void Assembler::andnpd(XMMRegister dst, XMMRegister src) {
8006 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8007 attributes.set_rex_vex_w_reverted();
8008 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8009 emit_int16(0x55, (0xC0 | encode));
8010 }
8011
8012 void Assembler::andps(XMMRegister dst, XMMRegister src) {
8013 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8014 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8015 emit_int16(0x54, (0xC0 | encode));
8016 }
8017
8018 void Assembler::andps(XMMRegister dst, Address src) {
8019 InstructionMark im(this);
8020 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8021 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8022 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8023 emit_int8(0x54);
8024 emit_operand(dst, src, 0);
8025 }
8026
8027 void Assembler::andpd(XMMRegister dst, Address src) {
8028 InstructionMark im(this);
8029 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8030 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8031 attributes.set_rex_vex_w_reverted();
8032 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8033 emit_int8(0x54);
8034 emit_operand(dst, src, 0);
8035 }
8036
8037 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8038 assert(VM_Version::supports_avx(), "");
8039 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8040 attributes.set_rex_vex_w_reverted();
8041 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8042 emit_int16(0x54, (0xC0 | encode));
8043 }
8044
8045 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8046 assert(VM_Version::supports_avx(), "");
8047 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8049 emit_int16(0x54, (0xC0 | encode));
8050 }
8051
8052 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8053 assert(VM_Version::supports_avx(), "");
8054 InstructionMark im(this);
8055 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8056 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8057 attributes.set_rex_vex_w_reverted();
8058 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8059 emit_int8(0x54);
8060 emit_operand(dst, src, 0);
8061 }
8062
8063 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8064 assert(VM_Version::supports_avx(), "");
8065 InstructionMark im(this);
8066 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8067 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8068 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8069 emit_int8(0x54);
8070 emit_operand(dst, src, 0);
8071 }
8072
8073 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
8074 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8075 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8076 attributes.set_rex_vex_w_reverted();
8077 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8078 emit_int16(0x56, (0xC0 | encode));
8079 }
8080
8081 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
8082 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8083 attributes.set_rex_vex_w_reverted();
8084 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8085 emit_int8(0x15);
8086 emit_int8((0xC0 | encode));
8087 }
8088
8089 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
8090 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8091 attributes.set_rex_vex_w_reverted();
8092 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8093 emit_int16(0x14, (0xC0 | encode));
8094 }
8095
8096 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
8097 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8098 attributes.set_rex_vex_w_reverted();
8099 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8100 emit_int16(0x57, (0xC0 | encode));
8101 }
8102
8103 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
8104 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8105 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8106 emit_int16(0x57, (0xC0 | encode));
8107 }
8108
8109 void Assembler::xorpd(XMMRegister dst, Address src) {
8110 InstructionMark im(this);
8111 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8112 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8113 attributes.set_rex_vex_w_reverted();
8114 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8115 emit_int8(0x57);
8116 emit_operand(dst, src, 0);
8117 }
8118
8119 void Assembler::xorps(XMMRegister dst, Address src) {
8120 InstructionMark im(this);
8121 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8122 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8123 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8124 emit_int8(0x57);
8125 emit_operand(dst, src, 0);
8126 }
8127
8128 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8129 assert(VM_Version::supports_avx(), "");
8130 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8131 attributes.set_rex_vex_w_reverted();
8132 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8133 emit_int16(0x57, (0xC0 | encode));
8134 }
8135
8136 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8137 assert(VM_Version::supports_avx(), "");
8138 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8139 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8140 emit_int16(0x57, (0xC0 | encode));
8141 }
8142
8143 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8144 assert(VM_Version::supports_avx(), "");
8145 InstructionMark im(this);
8146 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8147 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8148 attributes.set_rex_vex_w_reverted();
8149 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8150 emit_int8(0x57);
8151 emit_operand(dst, src, 0);
8152 }
8153
8154 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8155 assert(VM_Version::supports_avx(), "");
8156 InstructionMark im(this);
8157 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8158 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8159 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8160 emit_int8(0x57);
8161 emit_operand(dst, src, 0);
8162 }
8163
8164 // Integer vector arithmetic
8165 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8166 assert((VM_Version::supports_avx() && (vector_len == 0)) ||
8167 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
8168 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
8169 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8170 emit_int16(0x01, (0xC0 | encode));
8171 }
8172
8173 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8174 assert((VM_Version::supports_avx() && (vector_len == 0)) ||
8175 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
8176 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
8177 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8178 emit_int16(0x02, (0xC0 | encode));
8179 }
8180
8181 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
8182 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8183 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8184 emit_int16((unsigned char)0xFC, (0xC0 | encode));
8185 }
8186
8187 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
8188 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8189 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8190 emit_int16((unsigned char)0xFD, (0xC0 | encode));
8191 }
8192
8193 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
8194 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8195 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8196 emit_int16((unsigned char)0xFE, (0xC0 | encode));
8197 }
8198
8199 void Assembler::paddd(XMMRegister dst, Address src) {
8200 InstructionMark im(this);
8201 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8202 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8203 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8204 emit_int8((unsigned char)0xFE);
8205 emit_operand(dst, src, 0);
8206 }
8207
8208 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
8209 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8210 attributes.set_rex_vex_w_reverted();
8211 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8212 emit_int16((unsigned char)0xD4, (0xC0 | encode));
8213 }
8214
8215 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
8216 assert(VM_Version::supports_sse3(), "");
8217 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
8218 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8219 emit_int16(0x01, (0xC0 | encode));
8220 }
8221
8222 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
8223 assert(VM_Version::supports_sse3(), "");
8224 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
8225 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8226 emit_int16(0x02, (0xC0 | encode));
8227 }
8228
8229 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8230 assert(UseAVX > 0, "requires some form of AVX");
8231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8232 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8233 emit_int16((unsigned char)0xFC, (0xC0 | encode));
8234 }
8235
8236 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8237 assert(UseAVX > 0, "requires some form of AVX");
8238 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8240 emit_int16((unsigned char)0xFD, (0xC0 | encode));
8241 }
8242
8243 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8244 assert(UseAVX > 0, "requires some form of AVX");
8245 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8247 emit_int16((unsigned char)0xFE, (0xC0 | encode));
8248 }
8249
8250 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8251 assert(UseAVX > 0, "requires some form of AVX");
8252 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8253 attributes.set_rex_vex_w_reverted();
8254 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8255 emit_int16((unsigned char)0xD4, (0xC0 | encode));
8256 }
8257
8258 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8259 assert(UseAVX > 0, "requires some form of AVX");
8260 InstructionMark im(this);
8261 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8262 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8263 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8264 emit_int8((unsigned char)0xFC);
8265 emit_operand(dst, src, 0);
8266 }
8267
8268 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8269 assert(UseAVX > 0, "requires some form of AVX");
8270 InstructionMark im(this);
8271 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8272 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8273 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8274 emit_int8((unsigned char)0xFD);
8275 emit_operand(dst, src, 0);
8276 }
8277
8278 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8279 assert(UseAVX > 0, "requires some form of AVX");
8280 InstructionMark im(this);
8281 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8282 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8283 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8284 emit_int8((unsigned char)0xFE);
8285 emit_operand(dst, src, 0);
8286 }
8287
8288 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8289 assert(UseAVX > 0, "requires some form of AVX");
8290 InstructionMark im(this);
8291 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8292 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8293 attributes.set_rex_vex_w_reverted();
8294 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8295 emit_int8((unsigned char)0xD4);
8296 emit_operand(dst, src, 0);
8297 }
8298
8299 void Assembler::vaddsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8300 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8301 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8302 attributes.set_is_evex_instruction();
8303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8304 emit_int16(0x58, (0xC0 | encode));
8305 }
8306
8307 void Assembler::vsubsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8308 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8309 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8310 attributes.set_is_evex_instruction();
8311 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8312 emit_int16(0x5C, (0xC0 | encode));
8313 }
8314
8315 void Assembler::vdivsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8316 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8317 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8318 attributes.set_is_evex_instruction();
8319 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8320 emit_int16(0x5E, (0xC0 | encode));
8321 }
8322
8323 void Assembler::vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8324 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8325 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8326 attributes.set_is_evex_instruction();
8327 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8328 emit_int16(0x59, (0xC0 | encode));
8329 }
8330
8331 void Assembler::vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8332 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8333 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8334 attributes.set_is_evex_instruction();
8335 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8336 emit_int16(0x5F, (0xC0 | encode));
8337 }
8338
8339 void Assembler::eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
8340 assert(VM_Version::supports_avx10_2(), "");
8341 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8342 attributes.set_is_evex_instruction();
8343 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
8344 emit_int24(0x53, (0xC0 | encode), imm8);
8345 }
8346
8347 void Assembler::vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
8348 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8349 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8350 attributes.set_is_evex_instruction();
8351 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8352 emit_int16(0x5D, (0xC0 | encode));
8353 }
8354
8355 void Assembler::vsqrtsh(XMMRegister dst, XMMRegister src) {
8356 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
8357 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8358 attributes.set_is_evex_instruction();
8359 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
8360 emit_int16(0x51, (0xC0 | encode));
8361 }
8362
8363 void Assembler::vfmadd132sh(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
8364 assert(VM_Version::supports_avx512_fp16(), "");
8365 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8366 attributes.set_is_evex_instruction();
8367 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes);
8368 emit_int16((unsigned char)0x99, (0xC0 | encode));
8369 }
8370
8371 void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8372 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8373 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8374 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8375 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8376 emit_int16((unsigned char)0xEC, (0xC0 | encode));
8377 }
8378
8379 void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8380 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8381 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8382 InstructionMark im(this);
8383 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8384 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8385 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8386 emit_int8((unsigned char)0xEC);
8387 emit_operand(dst, src, 0);
8388 }
8389
8390 void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8391 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8392 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8393 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8394 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8395 emit_int16((unsigned char)0xED, (0xC0 | encode));
8396 }
8397
8398 void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8399 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8400 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8401 InstructionMark im(this);
8402 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8403 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8404 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8405 emit_int8((unsigned char)0xED);
8406 emit_operand(dst, src, 0);
8407 }
8408
8409 void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8410 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8411 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8412 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8413 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8414 emit_int16((unsigned char)0xDC, (0xC0 | encode));
8415 }
8416
8417 void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8418 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8419 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8420 InstructionMark im(this);
8421 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8422 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8423 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8424 emit_int8((unsigned char)0xDC);
8425 emit_operand(dst, src, 0);
8426 }
8427
8428
8429 void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8430 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8431 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8432 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8433 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8434 emit_int16((unsigned char)0xDD, (0xC0 | encode));
8435 }
8436
8437 void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8438 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8439 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8440 InstructionMark im(this);
8441 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8442 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8443 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8444 emit_int8((unsigned char)0xDD);
8445 emit_operand(dst, src, 0);
8446 }
8447
8448
8449 void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8450 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8451 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8452 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8453 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8454 emit_int16((unsigned char)0xE8, (0xC0 | encode));
8455 }
8456
8457 void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8458 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8459 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8460 InstructionMark im(this);
8461 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8462 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8463 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8464 emit_int8((unsigned char)0xE8);
8465 emit_operand(dst, src, 0);
8466 }
8467
8468 void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8469 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8470 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8471 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8472 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8473 emit_int16((unsigned char)0xE9, (0xC0 | encode));
8474 }
8475
8476 void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8477 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8478 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8479 InstructionMark im(this);
8480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8481 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8482 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8483 emit_int8((unsigned char)0xE9);
8484 emit_operand(dst, src, 0);
8485 }
8486
8487 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8488 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8489 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8490 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8491 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8492 emit_int16((unsigned char)0xD8, (0xC0 | encode));
8493 }
8494
8495 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8496 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8497 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8498 InstructionMark im(this);
8499 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8500 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8501 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8502 emit_int8((unsigned char)0xD8);
8503 emit_operand(dst, src, 0);
8504 }
8505
8506 void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8507 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8508 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8509 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8510 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8511 emit_int16((unsigned char)0xD9, (0xC0 | encode));
8512 }
8513
8514 void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8515 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8516 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8517 InstructionMark im(this);
8518 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8519 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8520 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8521 emit_int8((unsigned char)0xD9);
8522 emit_operand(dst, src, 0);
8523 }
8524
8525
8526 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
8527 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8528 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8529 emit_int16((unsigned char)0xF8, (0xC0 | encode));
8530 }
8531
8532 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
8533 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8534 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8535 emit_int16((unsigned char)0xF9, (0xC0 | encode));
8536 }
8537
8538 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
8539 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8540 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8541 emit_int16((unsigned char)0xFA, (0xC0 | encode));
8542 }
8543
8544 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
8545 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8546 attributes.set_rex_vex_w_reverted();
8547 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8548 emit_int8((unsigned char)0xFB);
8549 emit_int8((0xC0 | encode));
8550 }
8551
8552 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8553 assert(UseAVX > 0, "requires some form of AVX");
8554 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8555 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8556 emit_int16((unsigned char)0xF8, (0xC0 | encode));
8557 }
8558
8559 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8560 assert(UseAVX > 0, "requires some form of AVX");
8561 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8562 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8563 emit_int16((unsigned char)0xF9, (0xC0 | encode));
8564 }
8565
8566 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8567 assert(UseAVX > 0, "requires some form of AVX");
8568 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8569 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8570 emit_int16((unsigned char)0xFA, (0xC0 | encode));
8571 }
8572
8573 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8574 assert(UseAVX > 0, "requires some form of AVX");
8575 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8576 attributes.set_rex_vex_w_reverted();
8577 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8578 emit_int16((unsigned char)0xFB, (0xC0 | encode));
8579 }
8580
8581 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8582 assert(UseAVX > 0, "requires some form of AVX");
8583 InstructionMark im(this);
8584 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8585 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8586 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8587 emit_int8((unsigned char)0xF8);
8588 emit_operand(dst, src, 0);
8589 }
8590
8591 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8592 assert(UseAVX > 0, "requires some form of AVX");
8593 InstructionMark im(this);
8594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8595 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8596 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8597 emit_int8((unsigned char)0xF9);
8598 emit_operand(dst, src, 0);
8599 }
8600
8601 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8602 assert(UseAVX > 0, "requires some form of AVX");
8603 InstructionMark im(this);
8604 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8605 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8606 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8607 emit_int8((unsigned char)0xFA);
8608 emit_operand(dst, src, 0);
8609 }
8610
8611 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8612 assert(UseAVX > 0, "requires some form of AVX");
8613 InstructionMark im(this);
8614 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8615 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8616 attributes.set_rex_vex_w_reverted();
8617 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8618 emit_int8((unsigned char)0xFB);
8619 emit_operand(dst, src, 0);
8620 }
8621
8622 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
8623 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8624 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8625 emit_int16((unsigned char)0xD5, (0xC0 | encode));
8626 }
8627
8628 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
8629 assert(VM_Version::supports_sse4_1(), "");
8630 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8631 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8632 emit_int16(0x40, (0xC0 | encode));
8633 }
8634
8635 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
8636 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8637 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8638 emit_int16((unsigned char)0xF4, (0xC0 | encode));
8639 }
8640
8641 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8642 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
8643 (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
8644 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
8645 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8646 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8647 emit_int16((unsigned char)0xE4, (0xC0 | encode));
8648 }
8649
8650 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8651 assert(UseAVX > 0, "requires some form of AVX");
8652 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8654 emit_int16((unsigned char)0xD5, (0xC0 | encode));
8655 }
8656
8657 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8658 assert(UseAVX > 0, "requires some form of AVX");
8659 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8660 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8661 emit_int16(0x40, (0xC0 | encode));
8662 }
8663
8664 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8665 assert(UseAVX > 2, "requires some form of EVEX");
8666 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8667 attributes.set_is_evex_instruction();
8668 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8669 emit_int16(0x40, (0xC0 | encode));
8670 }
8671
8672 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8673 assert(UseAVX > 0, "requires some form of AVX");
8674 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8675 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8676 emit_int16((unsigned char)0xF4, (0xC0 | encode));
8677 }
8678
8679 void Assembler::vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8680 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8681 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
8682 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8683 attributes.set_rex_vex_w_reverted();
8684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8685 emit_int16(0x28, (0xC0 | encode));
8686 }
8687
8688 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8689 assert(UseAVX > 0, "requires some form of AVX");
8690 InstructionMark im(this);
8691 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8692 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8693 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8694 emit_int8((unsigned char)0xD5);
8695 emit_operand(dst, src, 0);
8696 }
8697
8698 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8699 assert(UseAVX > 0, "requires some form of AVX");
8700 InstructionMark im(this);
8701 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8702 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8703 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8704 emit_int8(0x40);
8705 emit_operand(dst, src, 0);
8706 }
8707
8708 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8709 assert(UseAVX > 2, "requires some form of EVEX");
8710 InstructionMark im(this);
8711 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
8712 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8713 attributes.set_is_evex_instruction();
8714 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8715 emit_int8(0x40);
8716 emit_operand(dst, src, 0);
8717 }
8718
8719 // Min, max
8720 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
8721 assert(VM_Version::supports_sse4_1(), "");
8722 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8723 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8724 emit_int16(0x38, (0xC0 | encode));
8725 }
8726
8727 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8728 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8729 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
8730 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8731 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8732 emit_int16(0x38, (0xC0 | encode));
8733 }
8734
8735 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
8736 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8737 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8738 emit_int16((unsigned char)0xEA, (0xC0 | encode));
8739 }
8740
8741 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8742 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8743 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
8744 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8745 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8746 emit_int16((unsigned char)0xEA, (0xC0 | encode));
8747 }
8748
8749 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
8750 assert(VM_Version::supports_sse4_1(), "");
8751 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8752 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8753 emit_int16(0x39, (0xC0 | encode));
8754 }
8755
8756 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8757 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8758 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
8759 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8760 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8761 emit_int16(0x39, (0xC0 | encode));
8762 }
8763
8764 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8765 assert(UseAVX > 2, "requires AVX512F");
8766 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8767 attributes.set_is_evex_instruction();
8768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8769 emit_int16(0x39, (0xC0 | encode));
8770 }
8771
8772 void Assembler::minps(XMMRegister dst, XMMRegister src) {
8773 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8774 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8775 emit_int16(0x5D, (0xC0 | encode));
8776 }
8777 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8778 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
8779 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8780 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8781 emit_int16(0x5D, (0xC0 | encode));
8782 }
8783
8784 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
8785 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8786 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8787 emit_int16(0x5D, (0xC0 | encode));
8788 }
8789 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8790 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
8791 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8792 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8793 emit_int16(0x5D, (0xC0 | encode));
8794 }
8795
8796 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
8797 assert(VM_Version::supports_sse4_1(), "");
8798 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8799 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8800 emit_int16(0x3C, (0xC0 | encode));
8801 }
8802
8803 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8804 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8805 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
8806 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8807 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8808 emit_int16(0x3C, (0xC0 | encode));
8809 }
8810
8811 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
8812 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8813 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8814 emit_int16((unsigned char)0xEE, (0xC0 | encode));
8815 }
8816
8817 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8818 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8819 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
8820 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8821 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8822 emit_int16((unsigned char)0xEE, (0xC0 | encode));
8823 }
8824
8825 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
8826 assert(VM_Version::supports_sse4_1(), "");
8827 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8828 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8829 emit_int16(0x3D, (0xC0 | encode));
8830 }
8831
8832 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8833 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8834 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
8835 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8837 emit_int16(0x3D, (0xC0 | encode));
8838 }
8839
8840 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8841 assert(UseAVX > 2, "requires AVX512F");
8842 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8843 attributes.set_is_evex_instruction();
8844 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8845 emit_int16(0x3D, (0xC0 | encode));
8846 }
8847
8848 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
8849 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8850 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8851 emit_int16(0x5F, (0xC0 | encode));
8852 }
8853
8854 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8855 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
8856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8857 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8858 emit_int16(0x5F, (0xC0 | encode));
8859 }
8860
8861 void Assembler::evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) {
8862 assert(VM_Version::supports_avx10_2(), "");
8863 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8864 attributes.set_is_evex_instruction();
8865 attributes.set_embedded_opmask_register_specifier(mask);
8866 if (merge) {
8867 attributes.reset_is_clear_context();
8868 }
8869 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8870 emit_int24(0x52, (0xC0 | encode), imm8);
8871 }
8872
8873 void Assembler::evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) {
8874 assert(VM_Version::supports_avx10_2(), "");
8875 InstructionMark im(this);
8876 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8877 attributes.set_is_evex_instruction();
8878 attributes.set_embedded_opmask_register_specifier(mask);
8879 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8880 if (merge) {
8881 attributes.reset_is_clear_context();
8882 }
8883 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8884 emit_int8(0x52);
8885 emit_operand(dst, src, 0);
8886 emit_int8(imm8);
8887 }
8888
8889 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
8890 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8891 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8892 emit_int16(0x5F, (0xC0 | encode));
8893 }
8894
8895 void Assembler::evminmaxpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) {
8896 assert(VM_Version::supports_avx10_2(), "");
8897 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8898 attributes.set_is_evex_instruction();
8899 attributes.set_embedded_opmask_register_specifier(mask);
8900 if (merge) {
8901 attributes.reset_is_clear_context();
8902 }
8903 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8904 emit_int24(0x52, (0xC0 | encode), imm8);
8905 }
8906
8907 void Assembler::evminmaxpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) {
8908 assert(VM_Version::supports_avx10_2(), "");
8909 InstructionMark im(this);
8910 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8911 attributes.set_is_evex_instruction();
8912 attributes.set_embedded_opmask_register_specifier(mask);
8913 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8914 if (merge) {
8915 attributes.reset_is_clear_context();
8916 }
8917 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8918 emit_int8(0x52);
8919 emit_operand(dst, src, 0);
8920 emit_int8(imm8);
8921 }
8922
8923 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8924 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
8925 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8926 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8927 emit_int16(0x5F, (0xC0 | encode));
8928 }
8929
8930 void Assembler::vpminub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8931 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8932 assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
8933 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8935 emit_int16((unsigned char)0xDA, (0xC0 | encode));
8936 }
8937
8938 void Assembler::vpminub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8939 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8940 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8941 InstructionMark im(this);
8942 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8943 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8944 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8945 emit_int8((unsigned char)0xDA);
8946 emit_operand(dst, src, 0);
8947 }
8948
8949 void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8950 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8951 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8952 attributes.set_is_evex_instruction();
8953 attributes.set_embedded_opmask_register_specifier(mask);
8954 if (merge) {
8955 attributes.reset_is_clear_context();
8956 }
8957 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8958 emit_int16((unsigned char)0xDA, (0xC0 | encode));
8959 }
8960
8961 void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8962 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8963 InstructionMark im(this);
8964 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8965 attributes.set_is_evex_instruction();
8966 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8967 attributes.set_embedded_opmask_register_specifier(mask);
8968 if (merge) {
8969 attributes.reset_is_clear_context();
8970 }
8971 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8972 emit_int8((unsigned char)0xDA);
8973 emit_operand(dst, src, 0);
8974 }
8975
8976 void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8977 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
8978 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8979 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8980 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8981 emit_int16(0x3A, (0xC0 | encode));
8982 }
8983
8984 void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8985 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
8986 assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
8987 InstructionMark im(this);
8988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8989 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
8990 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8991 emit_int8((unsigned char)0x3A);
8992 emit_operand(dst, src, 0);
8993 }
8994
8995 void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8996 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8997 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8998 attributes.set_is_evex_instruction();
8999 attributes.set_embedded_opmask_register_specifier(mask);
9000 if (merge) {
9001 attributes.reset_is_clear_context();
9002 }
9003 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9004 emit_int16(0x3A, (0xC0 | encode));
9005 }
9006
9007 void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9008 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9009 InstructionMark im(this);
9010 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9011 attributes.set_is_evex_instruction();
9012 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
9013 attributes.set_embedded_opmask_register_specifier(mask);
9014 if (merge) {
9015 attributes.reset_is_clear_context();
9016 }
9017 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9018 emit_int8(0x3A);
9019 emit_operand(dst, src, 0);
9020 }
9021
9022 void Assembler::vpminud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9023 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
9024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9025 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9026 emit_int16(0x3B, (0xC0 | encode));
9027 }
9028
9029 void Assembler::vpminud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9030 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
9031 InstructionMark im(this);
9032 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9033 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9034 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9035 emit_int8((unsigned char)0x3B);
9036 emit_operand(dst, src, 0);
9037 }
9038
9039 void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9040 assert(VM_Version::supports_evex(), "");
9041 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9042 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9043 attributes.set_is_evex_instruction();
9044 attributes.set_embedded_opmask_register_specifier(mask);
9045 if (merge) {
9046 attributes.reset_is_clear_context();
9047 }
9048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9049 emit_int16(0x3B, (0xC0 | encode));
9050 }
9051
9052 void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9053 assert(VM_Version::supports_evex(), "");
9054 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9055 InstructionMark im(this);
9056 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9057 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9058 attributes.set_is_evex_instruction();
9059 attributes.set_embedded_opmask_register_specifier(mask);
9060 if (merge) {
9061 attributes.reset_is_clear_context();
9062 }
9063 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9064 emit_int8(0x3B);
9065 emit_operand(dst, src, 0);
9066 }
9067
9068 void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9069 assert(VM_Version::supports_evex(), "");
9070 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9071 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9072 attributes.set_is_evex_instruction();
9073 attributes.set_embedded_opmask_register_specifier(mask);
9074 if (merge) {
9075 attributes.reset_is_clear_context();
9076 }
9077 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9078 emit_int16(0x3B, (0xC0 | encode));
9079 }
9080
9081 void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9082 assert(VM_Version::supports_evex(), "");
9083 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9084 InstructionMark im(this);
9085 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9086 attributes.set_is_evex_instruction();
9087 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9088 attributes.set_embedded_opmask_register_specifier(mask);
9089 if (merge) {
9090 attributes.reset_is_clear_context();
9091 }
9092 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9093 emit_int8(0x3B);
9094 emit_operand(dst, src, 0);
9095 }
9096
9097 void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9098 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9099 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
9100 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
9101 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9102 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9103 emit_int16((unsigned char)0xDE, (0xC0 | encode));
9104 }
9105
9106 void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9107 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9108 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
9109 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
9110 InstructionMark im(this);
9111 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9112 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
9113 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9114 emit_int8((unsigned char)0xDE);
9115 emit_operand(dst, src, 0);
9116 }
9117
9118 void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9119 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9120 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9121 attributes.set_is_evex_instruction();
9122 attributes.set_embedded_opmask_register_specifier(mask);
9123 if (merge) {
9124 attributes.reset_is_clear_context();
9125 }
9126 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9127 emit_int16((unsigned char)0xDE, (0xC0 | encode));
9128 }
9129
9130 void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9131 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9132 InstructionMark im(this);
9133 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9134 attributes.set_is_evex_instruction();
9135 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
9136 attributes.set_embedded_opmask_register_specifier(mask);
9137 if (merge) {
9138 attributes.reset_is_clear_context();
9139 }
9140 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9141 emit_int8((unsigned char)0xDE);
9142 emit_operand(dst, src, 0);
9143 }
9144
9145 void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9146 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9147 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
9148 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
9149 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9150 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9151 emit_int16(0x3E, (0xC0 | encode));
9152 }
9153
9154 void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9155 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9156 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
9157 assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
9158 InstructionMark im(this);
9159 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9160 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
9161 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9162 emit_int8((unsigned char)0x3E);
9163 emit_operand(dst, src, 0);
9164 }
9165
9166 void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9167 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9168 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9169 attributes.set_is_evex_instruction();
9170 attributes.set_embedded_opmask_register_specifier(mask);
9171 if (merge) {
9172 attributes.reset_is_clear_context();
9173 }
9174 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9175 emit_int16(0x3E, (0xC0 | encode));
9176 }
9177
9178 void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9179 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9180 InstructionMark im(this);
9181 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9182 attributes.set_is_evex_instruction();
9183 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
9184 attributes.set_embedded_opmask_register_specifier(mask);
9185 if (merge) {
9186 attributes.reset_is_clear_context();
9187 }
9188 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9189 emit_int8(0x3E);
9190 emit_operand(dst, src, 0);
9191 }
9192
9193 void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9194 assert(UseAVX > 0, "");
9195 assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
9196 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9198 emit_int16(0x3F, (0xC0 | encode));
9199 }
9200
9201 void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9202 assert(UseAVX > 0, "");
9203 assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
9204 InstructionMark im(this);
9205 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9206 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9207 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9208 emit_int8((unsigned char)0x3F);
9209 emit_operand(dst, src, 0);
9210 }
9211
9212 void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9213 assert(VM_Version::supports_evex(), "");
9214 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9216 attributes.set_is_evex_instruction();
9217 attributes.set_embedded_opmask_register_specifier(mask);
9218 if (merge) {
9219 attributes.reset_is_clear_context();
9220 }
9221 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9222 emit_int16(0x3F, (0xC0 | encode));
9223 }
9224
9225 void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9226 assert(VM_Version::supports_evex(), "");
9227 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9228 InstructionMark im(this);
9229 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9230 attributes.set_is_evex_instruction();
9231 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9232 attributes.set_embedded_opmask_register_specifier(mask);
9233 if (merge) {
9234 attributes.reset_is_clear_context();
9235 }
9236 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9237 emit_int8(0x3F);
9238 emit_operand(dst, src, 0);
9239 }
9240
9241 void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9242 assert(VM_Version::supports_evex(), "");
9243 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9244 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9245 attributes.set_is_evex_instruction();
9246 attributes.set_embedded_opmask_register_specifier(mask);
9247 if (merge) {
9248 attributes.reset_is_clear_context();
9249 }
9250 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9251 emit_int16(0x3F, (0xC0 | encode));
9252 }
9253
9254 void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9255 assert(VM_Version::supports_evex(), "");
9256 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9257 InstructionMark im(this);
9258 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9259 attributes.set_is_evex_instruction();
9260 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9261 attributes.set_embedded_opmask_register_specifier(mask);
9262 if (merge) {
9263 attributes.reset_is_clear_context();
9264 }
9265 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9266 emit_int8(0x3F);
9267 emit_operand(dst, src, 0);
9268 }
9269
9270 // Shift packed integers left by specified number of bits.
9271 void Assembler::psllw(XMMRegister dst, int shift) {
9272 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9273 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
9274 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9275 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9276 }
9277
9278 void Assembler::pslld(XMMRegister dst, int shift) {
9279 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9280 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
9281 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9282 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9283 }
9284
9285 void Assembler::psllq(XMMRegister dst, int shift) {
9286 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9287 attributes.set_rex_vex_w_reverted();
9288 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
9289 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9290 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9291 }
9292
9293 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
9294 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9295 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9296 emit_int16((unsigned char)0xF1, (0xC0 | encode));
9297 }
9298
9299 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
9300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9301 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9302 emit_int16((unsigned char)0xF2, (0xC0 | encode));
9303 }
9304
9305 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
9306 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9307 attributes.set_rex_vex_w_reverted();
9308 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9309 emit_int16((unsigned char)0xF3, (0xC0 | encode));
9310 }
9311
9312 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9313 assert(UseAVX > 0, "requires some form of AVX");
9314 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9315 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
9316 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9317 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9318 }
9319
9320 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9321 assert(UseAVX > 0, "requires some form of AVX");
9322 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9323 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
9324 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9325 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9326 }
9327
9328 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9329 assert(UseAVX > 0, "requires some form of AVX");
9330 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9331 attributes.set_rex_vex_w_reverted();
9332 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
9333 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9334 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9335 }
9336
9337 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9338 assert(UseAVX > 0, "requires some form of AVX");
9339 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9340 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9341 emit_int16((unsigned char)0xF1, (0xC0 | encode));
9342 }
9343
9344 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9345 assert(UseAVX > 0, "requires some form of AVX");
9346 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9347 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9348 emit_int16((unsigned char)0xF2, (0xC0 | encode));
9349 }
9350
9351 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9352 assert(UseAVX > 0, "requires some form of AVX");
9353 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9354 attributes.set_rex_vex_w_reverted();
9355 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9356 emit_int16((unsigned char)0xF3, (0xC0 | encode));
9357 }
9358
9359 // Shift packed integers logically right by specified number of bits.
9360 void Assembler::psrlw(XMMRegister dst, int shift) {
9361 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9362 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
9363 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9364 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9365 }
9366
9367 void Assembler::psrld(XMMRegister dst, int shift) {
9368 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9369 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
9370 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9371 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9372 }
9373
9374 void Assembler::psrlq(XMMRegister dst, int shift) {
9375 // Do not confuse it with psrldq SSE2 instruction which
9376 // shifts 128 bit value in xmm register by number of bytes.
9377 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9378 attributes.set_rex_vex_w_reverted();
9379 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
9380 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9381 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9382 }
9383
9384 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
9385 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9386 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9387 emit_int16((unsigned char)0xD1, (0xC0 | encode));
9388 }
9389
9390 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
9391 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9392 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9393 emit_int16((unsigned char)0xD2, (0xC0 | encode));
9394 }
9395
9396 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
9397 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9398 attributes.set_rex_vex_w_reverted();
9399 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9400 emit_int16((unsigned char)0xD3, (0xC0 | encode));
9401 }
9402
9403 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9404 assert(UseAVX > 0, "requires some form of AVX");
9405 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9406 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
9407 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9408 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9409 }
9410
9411 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9412 assert(UseAVX > 0, "requires some form of AVX");
9413 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9414 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
9415 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9416 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9417 }
9418
9419 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9420 assert(UseAVX > 0, "requires some form of AVX");
9421 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9422 attributes.set_rex_vex_w_reverted();
9423 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
9424 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9425 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9426 }
9427
9428 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9429 assert(UseAVX > 0, "requires some form of AVX");
9430 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9431 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9432 emit_int16((unsigned char)0xD1, (0xC0 | encode));
9433 }
9434
9435 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9436 assert(UseAVX > 0, "requires some form of AVX");
9437 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9438 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9439 emit_int16((unsigned char)0xD2, (0xC0 | encode));
9440 }
9441
9442 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9443 assert(UseAVX > 0, "requires some form of AVX");
9444 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9445 attributes.set_rex_vex_w_reverted();
9446 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9447 emit_int16((unsigned char)0xD3, (0xC0 | encode));
9448 }
9449
9450 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9451 assert(VM_Version::supports_avx512bw(), "");
9452 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9453 attributes.set_is_evex_instruction();
9454 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9455 emit_int16(0x10, (0xC0 | encode));
9456 }
9457
9458 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9459 assert(VM_Version::supports_avx512bw(), "");
9460 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9461 attributes.set_is_evex_instruction();
9462 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9463 emit_int16(0x12, (0xC0 | encode));
9464 }
9465
9466 // Shift packed integers arithmetically right by specified number of bits.
9467 void Assembler::psraw(XMMRegister dst, int shift) {
9468 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9469 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
9470 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9471 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9472 }
9473
9474 void Assembler::psrad(XMMRegister dst, int shift) {
9475 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9476 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
9477 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9478 emit_int8(0x72);
9479 emit_int8((0xC0 | encode));
9480 emit_int8(shift & 0xFF);
9481 }
9482
9483 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
9484 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9485 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9486 emit_int16((unsigned char)0xE1, (0xC0 | encode));
9487 }
9488
9489 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
9490 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9491 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9492 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9493 }
9494
9495 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9496 assert(UseAVX > 0, "requires some form of AVX");
9497 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9498 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
9499 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9500 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9501 }
9502
9503 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9504 assert(UseAVX > 0, "requires some form of AVX");
9505 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9506 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
9507 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9508 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9509 }
9510
9511 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9512 assert(UseAVX > 0, "requires some form of AVX");
9513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
9514 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9515 emit_int16((unsigned char)0xE1, (0xC0 | encode));
9516 }
9517
9518 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9519 assert(UseAVX > 0, "requires some form of AVX");
9520 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9521 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9522 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9523 }
9524
9525 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9526 assert(UseAVX > 2, "requires AVX512");
9527 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
9528 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9529 attributes.set_is_evex_instruction();
9530 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9531 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
9532 }
9533
9534 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9535 assert(UseAVX > 2, "requires AVX512");
9536 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
9537 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9538 attributes.set_is_evex_instruction();
9539 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9540 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9541 }
9542
9543 // logical operations packed integers
9544 void Assembler::pand(XMMRegister dst, XMMRegister src) {
9545 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9546 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9547 emit_int16((unsigned char)0xDB, (0xC0 | encode));
9548 }
9549
9550 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9551 assert(UseAVX > 0, "requires some form of AVX");
9552 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9553 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9554 emit_int16((unsigned char)0xDB, (0xC0 | encode));
9555 }
9556
9557 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9558 assert(UseAVX > 0, "requires some form of AVX");
9559 InstructionMark im(this);
9560 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9561 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9562 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9563 emit_int8((unsigned char)0xDB);
9564 emit_operand(dst, src, 0);
9565 }
9566
9567 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9568 evpandq(dst, k0, nds, src, false, vector_len);
9569 }
9570
9571 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9572 evpandq(dst, k0, nds, src, false, vector_len);
9573 }
9574
9575 //Variable Shift packed integers logically left.
9576 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9577 assert(UseAVX > 1, "requires AVX2");
9578 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9579 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9580 emit_int16(0x47, (0xC0 | encode));
9581 }
9582
9583 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9584 assert(UseAVX > 1, "requires AVX2");
9585 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9586 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9587 emit_int16(0x47, (0xC0 | encode));
9588 }
9589
9590 //Variable Shift packed integers logically right.
9591 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9592 assert(UseAVX > 1, "requires AVX2");
9593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9594 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9595 emit_int16(0x45, (0xC0 | encode));
9596 }
9597
9598 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9599 assert(UseAVX > 1, "requires AVX2");
9600 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9601 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9602 emit_int16(0x45, (0xC0 | encode));
9603 }
9604
9605 //Variable right Shift arithmetic packed integers .
9606 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9607 assert(UseAVX > 1, "requires AVX2");
9608 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9609 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9610 emit_int16(0x46, (0xC0 | encode));
9611 }
9612
9613 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9614 assert(VM_Version::supports_avx512bw(), "");
9615 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9616 attributes.set_is_evex_instruction();
9617 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9618 emit_int16(0x11, (0xC0 | encode));
9619 }
9620
9621 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9622 assert(UseAVX > 2, "requires AVX512");
9623 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
9624 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9625 attributes.set_is_evex_instruction();
9626 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9627 emit_int16(0x46, (0xC0 | encode));
9628 }
9629
9630 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9631 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
9632 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9633 attributes.set_is_evex_instruction();
9634 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9635 emit_int16(0x71, (0xC0 | encode));
9636 }
9637
9638 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9639 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
9640 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9641 attributes.set_is_evex_instruction();
9642 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9643 emit_int16(0x73, (0xC0 | encode));
9644 }
9645
9646 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
9647 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9648 attributes.set_rex_vex_w_reverted();
9649 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9650 emit_int16((unsigned char)0xDF, (0xC0 | encode));
9651 }
9652
9653 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9654 assert(UseAVX > 0, "requires some form of AVX");
9655 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9656 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9657 emit_int16((unsigned char)0xDF, (0xC0 | encode));
9658 }
9659
9660 void Assembler::por(XMMRegister dst, XMMRegister src) {
9661 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9663 emit_int16((unsigned char)0xEB, (0xC0 | encode));
9664 }
9665
9666 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9667 assert(UseAVX > 0, "requires some form of AVX");
9668 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9669 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9670 emit_int16((unsigned char)0xEB, (0xC0 | encode));
9671 }
9672
9673 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9674 assert(UseAVX > 0, "requires some form of AVX");
9675 InstructionMark im(this);
9676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9677 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9678 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9679 emit_int8((unsigned char)0xEB);
9680 emit_operand(dst, src, 0);
9681 }
9682
9683 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9684 evporq(dst, k0, nds, src, false, vector_len);
9685 }
9686
9687 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9688 evporq(dst, k0, nds, src, false, vector_len);
9689 }
9690
9691 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9692 assert(VM_Version::supports_evex(), "");
9693 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
9694 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9695 attributes.set_is_evex_instruction();
9696 attributes.set_embedded_opmask_register_specifier(mask);
9697 if (merge) {
9698 attributes.reset_is_clear_context();
9699 }
9700 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9701 emit_int16((unsigned char)0xEB, (0xC0 | encode));
9702 }
9703
9704 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9705 assert(VM_Version::supports_evex(), "");
9706 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
9707 InstructionMark im(this);
9708 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9709 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9710 attributes.set_is_evex_instruction();
9711 attributes.set_embedded_opmask_register_specifier(mask);
9712 if (merge) {
9713 attributes.reset_is_clear_context();
9714 }
9715 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9716 emit_int8((unsigned char)0xEB);
9717 emit_operand(dst, src, 0);
9718 }
9719
9720 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
9721 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9722 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9723 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9724 }
9725
9726 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9727 assert(UseAVX > 0, "requires some form of AVX");
9728 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9729 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
9730 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
9731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9732 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9733 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9734 }
9735
9736 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9737 assert(UseAVX > 0, "requires some form of AVX");
9738 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
9739 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
9740 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
9741 InstructionMark im(this);
9742 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9744 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9745 emit_int8((unsigned char)0xEF);
9746 emit_operand(dst, src, 0);
9747 }
9748
9749 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9750 assert(UseAVX > 2, "requires some form of EVEX");
9751 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9752 attributes.set_rex_vex_w_reverted();
9753 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9754 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9755 }
9756
9757 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9758 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
9759 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9760 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9761 attributes.set_is_evex_instruction();
9762 attributes.set_embedded_opmask_register_specifier(mask);
9763 if (merge) {
9764 attributes.reset_is_clear_context();
9765 }
9766 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9767 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9768 }
9769
9770 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9771 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9772 InstructionMark im(this);
9773 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9774 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
9775 attributes.set_is_evex_instruction();
9776 attributes.set_embedded_opmask_register_specifier(mask);
9777 if (merge) {
9778 attributes.reset_is_clear_context();
9779 }
9780 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9781 emit_int8((unsigned char)0xEF);
9782 emit_operand(dst, src, 0);
9783 }
9784
9785 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9786 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
9787 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9788 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9789 attributes.set_is_evex_instruction();
9790 attributes.set_embedded_opmask_register_specifier(mask);
9791 if (merge) {
9792 attributes.reset_is_clear_context();
9793 }
9794 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9795 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9796 }
9797
9798 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9799 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9800 InstructionMark im(this);
9801 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9802 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
9803 attributes.set_is_evex_instruction();
9804 attributes.set_embedded_opmask_register_specifier(mask);
9805 if (merge) {
9806 attributes.reset_is_clear_context();
9807 }
9808 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9809 emit_int8((unsigned char)0xEF);
9810 emit_operand(dst, src, 0);
9811 }
9812
9813 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9814 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9815 InstructionMark im(this);
9816 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9817 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
9818 attributes.set_is_evex_instruction();
9819 attributes.set_embedded_opmask_register_specifier(mask);
9820 if (merge) {
9821 attributes.reset_is_clear_context();
9822 }
9823 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9824 emit_int8((unsigned char)0xDB);
9825 emit_operand(dst, src, 0);
9826 }
9827
9828 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9829 assert(VM_Version::supports_evex(), "requires AVX512F");
9830 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
9831 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9832 attributes.set_is_evex_instruction();
9833 attributes.set_embedded_opmask_register_specifier(mask);
9834 if (merge) {
9835 attributes.reset_is_clear_context();
9836 }
9837 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9838 emit_int16((unsigned char)0xDB, (0xC0 | encode));
9839 }
9840
9841 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9842 assert(VM_Version::supports_evex(), "requires AVX512F");
9843 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
9844 InstructionMark im(this);
9845 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9846 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
9847 attributes.set_is_evex_instruction();
9848 attributes.set_embedded_opmask_register_specifier(mask);
9849 if (merge) {
9850 attributes.reset_is_clear_context();
9851 }
9852 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9853 emit_int8((unsigned char)0xDB);
9854 emit_operand(dst, src, 0);
9855 }
9856
9857 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9858 assert(VM_Version::supports_evex(), "requires AVX512F");
9859 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
9860 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9861 attributes.set_is_evex_instruction();
9862 attributes.set_embedded_opmask_register_specifier(mask);
9863 if (merge) {
9864 attributes.reset_is_clear_context();
9865 }
9866 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9867 emit_int16((unsigned char)0xEB, (0xC0 | encode));
9868 }
9869
9870 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9871 assert(VM_Version::supports_evex(), "requires AVX512F");
9872 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
9873 InstructionMark im(this);
9874 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9875 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
9876 attributes.set_is_evex_instruction();
9877 attributes.set_embedded_opmask_register_specifier(mask);
9878 if (merge) {
9879 attributes.reset_is_clear_context();
9880 }
9881 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9882 emit_int8((unsigned char)0xEB);
9883 emit_operand(dst, src, 0);
9884 }
9885
9886 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
9887 assert(VM_Version::supports_evex(), "requires EVEX support");
9888 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9889 attributes.set_is_evex_instruction();
9890 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9891 emit_int16((unsigned char)0xEF, (0xC0 | encode));
9892 }
9893
9894 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
9895 assert(VM_Version::supports_evex(), "requires EVEX support");
9896 assert(dst != xnoreg, "sanity");
9897 InstructionMark im(this);
9898 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9899 attributes.set_is_evex_instruction();
9900 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
9901 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9902 emit_int8((unsigned char)0xEF);
9903 emit_operand(dst, src, 0);
9904 }
9905
9906 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9907 assert(VM_Version::supports_evex(), "requires EVEX support");
9908 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9909 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9910 attributes.set_is_evex_instruction();
9911 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9912 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9913 }
9914
9915 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9916 assert(VM_Version::supports_evex(), "requires EVEX support");
9917 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9918 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9919 attributes.set_is_evex_instruction();
9920 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9921 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9922 }
9923
9924 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9925 assert(VM_Version::supports_evex(), "requires EVEX support");
9926 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9927 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9928 attributes.set_is_evex_instruction();
9929 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9930 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9931 }
9932
9933 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
9934 assert(VM_Version::supports_evex(), "requires EVEX support");
9935 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9936 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9937 attributes.set_is_evex_instruction();
9938 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9939 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9940 }
9941
9942 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9943 assert(VM_Version::supports_evex(), "requires EVEX support");
9944 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9945 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9946 attributes.set_is_evex_instruction();
9947 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9948 emit_int16(0x15, (unsigned char)(0xC0 | encode));
9949 }
9950
9951 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9952 assert(VM_Version::supports_evex(), "requires EVEX support");
9953 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9954 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9955 attributes.set_is_evex_instruction();
9956 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9957 emit_int16(0x15, (unsigned char)(0xC0 | encode));
9958 }
9959
9960 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9961 assert(VM_Version::supports_evex(), "requires EVEX support");
9962 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9963 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9964 attributes.set_is_evex_instruction();
9965 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9966 emit_int16(0x14, (unsigned char)(0xC0 | encode));
9967 }
9968
9969 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
9970 assert(VM_Version::supports_evex(), "requires EVEX support");
9971 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
9972 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
9973 attributes.set_is_evex_instruction();
9974 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9975 emit_int16(0x14, (unsigned char)(0xC0 | encode));
9976 }
9977
9978 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9979 assert(VM_Version::supports_avx512cd(), "");
9980 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9981 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9982 attributes.set_is_evex_instruction();
9983 attributes.set_embedded_opmask_register_specifier(mask);
9984 if (merge) {
9985 attributes.reset_is_clear_context();
9986 }
9987 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9988 emit_int16(0x44, (0xC0 | encode));
9989 }
9990
9991 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9992 assert(VM_Version::supports_avx512cd(), "");
9993 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9994 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9995 attributes.set_is_evex_instruction();
9996 attributes.set_embedded_opmask_register_specifier(mask);
9997 if (merge) {
9998 attributes.reset_is_clear_context();
9999 }
10000 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10001 emit_int16(0x44, (0xC0 | encode));
10002 }
10003
10004 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
10005 assert(VM_Version::supports_evex(), "requires EVEX support");
10006 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10007 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10008 attributes.set_is_evex_instruction();
10009 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10010 emit_int8(0x25);
10011 emit_int8((unsigned char)(0xC0 | encode));
10012 emit_int8(imm8);
10013 }
10014
10015 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
10016 assert(VM_Version::supports_evex(), "requires EVEX support");
10017 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10018 assert(dst != xnoreg, "sanity");
10019 InstructionMark im(this);
10020 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10021 attributes.set_is_evex_instruction();
10022 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
10023 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10024 emit_int8(0x25);
10025 emit_operand(dst, src3, 1);
10026 emit_int8(imm8);
10027 }
10028
10029 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
10030 assert(VM_Version::supports_evex(), "requires AVX512F");
10031 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
10032 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10033 attributes.set_is_evex_instruction();
10034 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10035 emit_int8(0x25);
10036 emit_int8((unsigned char)(0xC0 | encode));
10037 emit_int8(imm8);
10038 }
10039
10040 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
10041 assert(VM_Version::supports_evex(), "requires EVEX support");
10042 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10043 assert(dst != xnoreg, "sanity");
10044 InstructionMark im(this);
10045 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10046 attributes.set_is_evex_instruction();
10047 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
10048 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10049 emit_int8(0x25);
10050 emit_operand(dst, src3, 1);
10051 emit_int8(imm8);
10052 }
10053
10054 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10055 assert(VM_Version::supports_evex(), "");
10056 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10057 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10058 attributes.set_is_evex_instruction();
10059 attributes.set_embedded_opmask_register_specifier(mask);
10060 if (merge) {
10061 attributes.reset_is_clear_context();
10062 }
10063 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10064 emit_int16((unsigned char)0x88, (0xC0 | encode));
10065 }
10066
10067 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10068 assert(VM_Version::supports_evex(), "");
10069 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10070 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10071 attributes.set_is_evex_instruction();
10072 attributes.set_embedded_opmask_register_specifier(mask);
10073 if (merge) {
10074 attributes.reset_is_clear_context();
10075 }
10076 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10077 emit_int16((unsigned char)0x88, (0xC0 | encode));
10078 }
10079
10080 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10081 assert(VM_Version::supports_avx512_vbmi2(), "");
10082 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10083 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10084 attributes.set_is_evex_instruction();
10085 attributes.set_embedded_opmask_register_specifier(mask);
10086 if (merge) {
10087 attributes.reset_is_clear_context();
10088 }
10089 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10090 emit_int16(0x62, (0xC0 | encode));
10091 }
10092
10093 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10094 assert(VM_Version::supports_avx512_vbmi2(), "");
10095 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10096 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10097 attributes.set_is_evex_instruction();
10098 attributes.set_embedded_opmask_register_specifier(mask);
10099 if (merge) {
10100 attributes.reset_is_clear_context();
10101 }
10102 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10103 emit_int16(0x62, (0xC0 | encode));
10104 }
10105
10106 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10107 assert(VM_Version::supports_evex(), "");
10108 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10109 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10110 attributes.set_is_evex_instruction();
10111 attributes.set_embedded_opmask_register_specifier(mask);
10112 if (merge) {
10113 attributes.reset_is_clear_context();
10114 }
10115 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10116 emit_int16((unsigned char)0x89, (0xC0 | encode));
10117 }
10118
10119 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
10120 assert(VM_Version::supports_evex(), "");
10121 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10122 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10123 attributes.set_is_evex_instruction();
10124 attributes.set_embedded_opmask_register_specifier(mask);
10125 if (merge) {
10126 attributes.reset_is_clear_context();
10127 }
10128 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10129 emit_int16((unsigned char)0x89, (0xC0 | encode));
10130 }
10131
10132 // vinserti forms
10133
10134 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10135 assert(VM_Version::supports_avx2(), "");
10136 assert(imm8 <= 0x01, "imm8: %u", imm8);
10137 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10138 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10139 // last byte:
10140 // 0x00 - insert into lower 128 bits
10141 // 0x01 - insert into upper 128 bits
10142 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
10143 }
10144
10145 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
10146 assert(VM_Version::supports_avx2(), "");
10147 assert(dst != xnoreg, "sanity");
10148 assert(imm8 <= 0x01, "imm8: %u", imm8);
10149 InstructionMark im(this);
10150 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10151 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10152 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10153 emit_int8(0x38);
10154 emit_operand(dst, src, 1);
10155 // 0x00 - insert into lower 128 bits
10156 // 0x01 - insert into upper 128 bits
10157 emit_int8(imm8 & 0x01);
10158 }
10159
10160 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10161 assert(VM_Version::supports_evex(), "");
10162 assert(imm8 <= 0x03, "imm8: %u", imm8);
10163 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10164 attributes.set_is_evex_instruction();
10165 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10166 // imm8:
10167 // 0x00 - insert into q0 128 bits (0..127)
10168 // 0x01 - insert into q1 128 bits (128..255)
10169 // 0x02 - insert into q2 128 bits (256..383)
10170 // 0x03 - insert into q3 128 bits (384..511)
10171 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
10172 }
10173
10174 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
10175 assert(VM_Version::supports_evex(), "");
10176 assert(dst != xnoreg, "sanity");
10177 assert(imm8 <= 0x03, "imm8: %u", imm8);
10178 InstructionMark im(this);
10179 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10180 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10181 attributes.set_is_evex_instruction();
10182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10183 emit_int8(0x18);
10184 emit_operand(dst, src, 1);
10185 // 0x00 - insert into q0 128 bits (0..127)
10186 // 0x01 - insert into q1 128 bits (128..255)
10187 // 0x02 - insert into q2 128 bits (256..383)
10188 // 0x03 - insert into q3 128 bits (384..511)
10189 emit_int8(imm8 & 0x03);
10190 }
10191
10192 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10193 assert(VM_Version::supports_evex(), "");
10194 assert(imm8 <= 0x01, "imm8: %u", imm8);
10195 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10196 attributes.set_is_evex_instruction();
10197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10198 //imm8:
10199 // 0x00 - insert into lower 256 bits
10200 // 0x01 - insert into upper 256 bits
10201 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
10202 }
10203
10204 void Assembler::evinserti64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8, int vector_len) {
10205 assert(VM_Version::supports_avx512dq(), "");
10206 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10207 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10208 attributes.set_is_evex_instruction();
10209 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10210 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
10211 }
10212
10213
10214 // vinsertf forms
10215
10216 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10217 assert(VM_Version::supports_avx(), "");
10218 assert(imm8 <= 0x01, "imm8: %u", imm8);
10219 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10220 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10221 // imm8:
10222 // 0x00 - insert into lower 128 bits
10223 // 0x01 - insert into upper 128 bits
10224 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
10225 }
10226
10227 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
10228 assert(VM_Version::supports_avx(), "");
10229 assert(dst != xnoreg, "sanity");
10230 assert(imm8 <= 0x01, "imm8: %u", imm8);
10231 InstructionMark im(this);
10232 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10233 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10234 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10235 emit_int8(0x18);
10236 emit_operand(dst, src, 1);
10237 // 0x00 - insert into lower 128 bits
10238 // 0x01 - insert into upper 128 bits
10239 emit_int8(imm8 & 0x01);
10240 }
10241
10242 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10243 assert(VM_Version::supports_evex(), "");
10244 assert(imm8 <= 0x03, "imm8: %u", imm8);
10245 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10247 // imm8:
10248 // 0x00 - insert into q0 128 bits (0..127)
10249 // 0x01 - insert into q1 128 bits (128..255)
10250 // 0x02 - insert into q0 128 bits (256..383)
10251 // 0x03 - insert into q1 128 bits (384..512)
10252 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
10253 }
10254
10255 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
10256 assert(VM_Version::supports_evex(), "");
10257 assert(dst != xnoreg, "sanity");
10258 assert(imm8 <= 0x03, "imm8: %u", imm8);
10259 InstructionMark im(this);
10260 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10261 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10262 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10263 emit_int8(0x18);
10264 emit_operand(dst, src, 1);
10265 // 0x00 - insert into q0 128 bits (0..127)
10266 // 0x01 - insert into q1 128 bits (128..255)
10267 // 0x02 - insert into q0 128 bits (256..383)
10268 // 0x03 - insert into q1 128 bits (384..512)
10269 emit_int8(imm8 & 0x03);
10270 }
10271
10272 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
10273 assert(VM_Version::supports_evex(), "");
10274 assert(imm8 <= 0x01, "imm8: %u", imm8);
10275 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10276 attributes.set_is_evex_instruction();
10277 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10278 // imm8:
10279 // 0x00 - insert into lower 256 bits
10280 // 0x01 - insert into upper 256 bits
10281 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
10282 }
10283
10284 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
10285 assert(VM_Version::supports_evex(), "");
10286 assert(dst != xnoreg, "sanity");
10287 assert(imm8 <= 0x01, "imm8: %u", imm8);
10288 InstructionMark im(this);
10289 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10290 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
10291 attributes.set_is_evex_instruction();
10292 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10293 emit_int8(0x1A);
10294 emit_operand(dst, src, 1);
10295 // 0x00 - insert into lower 256 bits
10296 // 0x01 - insert into upper 256 bits
10297 emit_int8(imm8 & 0x01);
10298 }
10299
10300
10301 // vextracti forms
10302
10303 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10304 assert(VM_Version::supports_avx2(), "");
10305 assert(imm8 <= 0x01, "imm8: %u", imm8);
10306 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10307 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10308 // imm8:
10309 // 0x00 - extract from lower 128 bits
10310 // 0x01 - extract from upper 128 bits
10311 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
10312 }
10313
10314 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
10315 assert(VM_Version::supports_avx2(), "");
10316 assert(src != xnoreg, "sanity");
10317 assert(imm8 <= 0x01, "imm8: %u", imm8);
10318 InstructionMark im(this);
10319 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10320 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10321 attributes.reset_is_clear_context();
10322 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10323 emit_int8(0x39);
10324 emit_operand(src, dst, 1);
10325 // 0x00 - extract from lower 128 bits
10326 // 0x01 - extract from upper 128 bits
10327 emit_int8(imm8 & 0x01);
10328 }
10329
10330 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10331 assert(VM_Version::supports_evex(), "");
10332 assert(imm8 <= 0x03, "imm8: %u", imm8);
10333 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10334 attributes.set_is_evex_instruction();
10335 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10336 // imm8:
10337 // 0x00 - extract from bits 127:0
10338 // 0x01 - extract from bits 255:128
10339 // 0x02 - extract from bits 383:256
10340 // 0x03 - extract from bits 511:384
10341 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
10342 }
10343
10344 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
10345 assert(VM_Version::supports_evex(), "");
10346 assert(src != xnoreg, "sanity");
10347 assert(imm8 <= 0x03, "imm8: %u", imm8);
10348 InstructionMark im(this);
10349 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10350 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10351 attributes.reset_is_clear_context();
10352 attributes.set_is_evex_instruction();
10353 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10354 emit_int8(0x39);
10355 emit_operand(src, dst, 1);
10356 // 0x00 - extract from bits 127:0
10357 // 0x01 - extract from bits 255:128
10358 // 0x02 - extract from bits 383:256
10359 // 0x03 - extract from bits 511:384
10360 emit_int8(imm8 & 0x03);
10361 }
10362
10363 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10364 assert(VM_Version::supports_avx512dq(), "");
10365 assert(imm8 <= 0x03, "imm8: %u", imm8);
10366 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10367 attributes.set_is_evex_instruction();
10368 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10369 // imm8:
10370 // 0x00 - extract from bits 127:0
10371 // 0x01 - extract from bits 255:128
10372 // 0x02 - extract from bits 383:256
10373 // 0x03 - extract from bits 511:384
10374 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
10375 }
10376
10377 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10378 assert(VM_Version::supports_evex(), "");
10379 assert(imm8 <= 0x01, "imm8: %u", imm8);
10380 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10381 attributes.set_is_evex_instruction();
10382 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10383 // imm8:
10384 // 0x00 - extract from lower 256 bits
10385 // 0x01 - extract from upper 256 bits
10386 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
10387 }
10388
10389 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
10390 assert(VM_Version::supports_evex(), "");
10391 assert(src != xnoreg, "sanity");
10392 assert(imm8 <= 0x01, "imm8: %u", imm8);
10393 InstructionMark im(this);
10394 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10395 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
10396 attributes.reset_is_clear_context();
10397 attributes.set_is_evex_instruction();
10398 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10399 emit_int8(0x38);
10400 emit_operand(src, dst, 1);
10401 // 0x00 - extract from lower 256 bits
10402 // 0x01 - extract from upper 256 bits
10403 emit_int8(imm8 & 0x01);
10404 }
10405 // vextractf forms
10406
10407 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10408 assert(VM_Version::supports_avx(), "");
10409 assert(imm8 <= 0x01, "imm8: %u", imm8);
10410 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10411 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10412 // imm8:
10413 // 0x00 - extract from lower 128 bits
10414 // 0x01 - extract from upper 128 bits
10415 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
10416 }
10417
10418 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
10419 assert(VM_Version::supports_avx(), "");
10420 assert(src != xnoreg, "sanity");
10421 assert(imm8 <= 0x01, "imm8: %u", imm8);
10422 InstructionMark im(this);
10423 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10424 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10425 attributes.reset_is_clear_context();
10426 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10427 emit_int8(0x19);
10428 emit_operand(src, dst, 1);
10429 // 0x00 - extract from lower 128 bits
10430 // 0x01 - extract from upper 128 bits
10431 emit_int8(imm8 & 0x01);
10432 }
10433
10434 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10435 assert(VM_Version::supports_evex(), "");
10436 assert(imm8 <= 0x03, "imm8: %u", imm8);
10437 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10438 attributes.set_is_evex_instruction();
10439 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10440 // imm8:
10441 // 0x00 - extract from bits 127:0
10442 // 0x01 - extract from bits 255:128
10443 // 0x02 - extract from bits 383:256
10444 // 0x03 - extract from bits 511:384
10445 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
10446 }
10447
10448 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
10449 assert(VM_Version::supports_evex(), "");
10450 assert(src != xnoreg, "sanity");
10451 assert(imm8 <= 0x03, "imm8: %u", imm8);
10452 InstructionMark im(this);
10453 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10454 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10455 attributes.reset_is_clear_context();
10456 attributes.set_is_evex_instruction();
10457 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10458 emit_int8(0x19);
10459 emit_operand(src, dst, 1);
10460 // 0x00 - extract from bits 127:0
10461 // 0x01 - extract from bits 255:128
10462 // 0x02 - extract from bits 383:256
10463 // 0x03 - extract from bits 511:384
10464 emit_int8(imm8 & 0x03);
10465 }
10466
10467 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10468 assert(VM_Version::supports_avx512dq(), "");
10469 assert(imm8 <= 0x03, "imm8: %u", imm8);
10470 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10471 attributes.set_is_evex_instruction();
10472 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10473 // imm8:
10474 // 0x00 - extract from bits 127:0
10475 // 0x01 - extract from bits 255:128
10476 // 0x02 - extract from bits 383:256
10477 // 0x03 - extract from bits 511:384
10478 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
10479 }
10480
10481 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
10482 assert(VM_Version::supports_evex(), "");
10483 assert(imm8 <= 0x01, "imm8: %u", imm8);
10484 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10485 attributes.set_is_evex_instruction();
10486 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10487 // imm8:
10488 // 0x00 - extract from lower 256 bits
10489 // 0x01 - extract from upper 256 bits
10490 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
10491 }
10492
10493 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
10494 assert(VM_Version::supports_evex(), "");
10495 assert(src != xnoreg, "sanity");
10496 assert(imm8 <= 0x01, "imm8: %u", imm8);
10497 InstructionMark im(this);
10498 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10499 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
10500 attributes.reset_is_clear_context();
10501 attributes.set_is_evex_instruction();
10502 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10503 emit_int8(0x1B);
10504 emit_operand(src, dst, 1);
10505 // 0x00 - extract from lower 256 bits
10506 // 0x01 - extract from upper 256 bits
10507 emit_int8(imm8 & 0x01);
10508 }
10509
10510 void Assembler::extractps(Register dst, XMMRegister src, uint8_t imm8) {
10511 assert(VM_Version::supports_sse4_1(), "");
10512 assert(imm8 <= 0x03, "imm8: %u", imm8);
10513 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10514 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes, true);
10515 // imm8:
10516 // 0x00 - extract from bits 31:0
10517 // 0x01 - extract from bits 63:32
10518 // 0x02 - extract from bits 95:64
10519 // 0x03 - extract from bits 127:96
10520 emit_int24(0x17, (0xC0 | encode), imm8 & 0x03);
10521 }
10522
10523 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10524 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
10525 assert(VM_Version::supports_avx2(), "");
10526 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10527 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10528 emit_int16(0x78, (0xC0 | encode));
10529 }
10530
10531 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
10532 assert(VM_Version::supports_avx2(), "");
10533 assert(dst != xnoreg, "sanity");
10534 InstructionMark im(this);
10535 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10536 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
10537 // swap src<->dst for encoding
10538 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10539 emit_int8(0x78);
10540 emit_operand(dst, src, 0);
10541 }
10542
10543 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10544 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
10545 assert(VM_Version::supports_avx2(), "");
10546 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10547 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10548 emit_int16(0x79, (0xC0 | encode));
10549 }
10550
10551 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
10552 assert(VM_Version::supports_avx2(), "");
10553 assert(dst != xnoreg, "sanity");
10554 InstructionMark im(this);
10555 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10556 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
10557 // swap src<->dst for encoding
10558 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10559 emit_int8(0x79);
10560 emit_operand(dst, src, 0);
10561 }
10562
10563 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10564 assert(UseAVX > 0, "requires some form of AVX");
10565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10566 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10567 emit_int16((unsigned char)0xF6, (0xC0 | encode));
10568 }
10569
10570 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10571 assert(UseAVX > 0, "requires some form of AVX");
10572 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10573 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10574 emit_int16(0x69, (0xC0 | encode));
10575 }
10576
10577 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10578 assert(UseAVX > 0, "requires some form of AVX");
10579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10580 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10581 emit_int16(0x61, (0xC0 | encode));
10582 }
10583
10584 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10585 assert(UseAVX > 0, "requires some form of AVX");
10586 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10587 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10588 emit_int16(0x6A, (0xC0 | encode));
10589 }
10590
10591 void Assembler::vpunpckhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10592 assert(UseAVX > 0, "requires some form of AVX");
10593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10594 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10595 emit_int16(0x6D, (0xC0 | encode));
10596 }
10597
10598 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10599 assert(UseAVX > 0, "requires some form of AVX");
10600 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10601 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10602 emit_int16(0x62, (0xC0 | encode));
10603 }
10604
10605 void Assembler::vpunpcklqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
10606 assert(UseAVX > 0, "requires some form of AVX");
10607 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10608 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10609 emit_int16(0x6C, (0xC0 | encode));
10610 }
10611
10612 // xmm/mem sourced byte/word/dword/qword replicate
10613 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10614 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10615 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10616 attributes.set_is_evex_instruction();
10617 attributes.set_embedded_opmask_register_specifier(mask);
10618 if (merge) {
10619 attributes.reset_is_clear_context();
10620 }
10621 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10622 emit_int16((unsigned char)0xFC, (0xC0 | encode));
10623 }
10624
10625 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10626 InstructionMark im(this);
10627 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10628 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10629 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10630 attributes.set_is_evex_instruction();
10631 attributes.set_embedded_opmask_register_specifier(mask);
10632 if (merge) {
10633 attributes.reset_is_clear_context();
10634 }
10635 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10636 emit_int8((unsigned char)0xFC);
10637 emit_operand(dst, src, 0);
10638 }
10639
10640 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10641 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10642 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10643 attributes.set_is_evex_instruction();
10644 attributes.set_embedded_opmask_register_specifier(mask);
10645 if (merge) {
10646 attributes.reset_is_clear_context();
10647 }
10648 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10649 emit_int16((unsigned char)0xFD, (0xC0 | encode));
10650 }
10651
10652 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10653 InstructionMark im(this);
10654 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10655 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10656 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10657 attributes.set_is_evex_instruction();
10658 attributes.set_embedded_opmask_register_specifier(mask);
10659 if (merge) {
10660 attributes.reset_is_clear_context();
10661 }
10662 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10663 emit_int8((unsigned char)0xFD);
10664 emit_operand(dst, src, 0);
10665 }
10666
10667 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10668 assert(VM_Version::supports_evex(), "");
10669 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10670 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10671 attributes.set_is_evex_instruction();
10672 attributes.set_embedded_opmask_register_specifier(mask);
10673 if (merge) {
10674 attributes.reset_is_clear_context();
10675 }
10676 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10677 emit_int16((unsigned char)0xFE, (0xC0 | encode));
10678 }
10679
10680 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10681 InstructionMark im(this);
10682 assert(VM_Version::supports_evex(), "");
10683 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10684 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10685 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10686 attributes.set_is_evex_instruction();
10687 attributes.set_embedded_opmask_register_specifier(mask);
10688 if (merge) {
10689 attributes.reset_is_clear_context();
10690 }
10691 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10692 emit_int8((unsigned char)0xFE);
10693 emit_operand(dst, src, 0);
10694 }
10695
10696 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10697 assert(VM_Version::supports_evex(), "");
10698 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10699 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10700 attributes.set_is_evex_instruction();
10701 attributes.set_embedded_opmask_register_specifier(mask);
10702 if (merge) {
10703 attributes.reset_is_clear_context();
10704 }
10705 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10706 emit_int16((unsigned char)0xD4, (0xC0 | encode));
10707 }
10708
10709 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10710 InstructionMark im(this);
10711 assert(VM_Version::supports_evex(), "");
10712 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10713 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10714 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10715 attributes.set_is_evex_instruction();
10716 attributes.set_embedded_opmask_register_specifier(mask);
10717 if (merge) {
10718 attributes.reset_is_clear_context();
10719 }
10720 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10721 emit_int8((unsigned char)0xD4);
10722 emit_operand(dst, src, 0);
10723 }
10724
10725 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10726 assert(VM_Version::supports_evex(), "");
10727 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10728 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10729 attributes.set_is_evex_instruction();
10730 attributes.set_embedded_opmask_register_specifier(mask);
10731 if (merge) {
10732 attributes.reset_is_clear_context();
10733 }
10734 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10735 emit_int16(0x58, (0xC0 | encode));
10736 }
10737
10738 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10739 InstructionMark im(this);
10740 assert(VM_Version::supports_evex(), "");
10741 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10742 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10743 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10744 attributes.set_is_evex_instruction();
10745 attributes.set_embedded_opmask_register_specifier(mask);
10746 if (merge) {
10747 attributes.reset_is_clear_context();
10748 }
10749 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10750 emit_int8(0x58);
10751 emit_operand(dst, src, 0);
10752 }
10753
10754 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10755 assert(VM_Version::supports_evex(), "");
10756 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10757 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10758 attributes.set_is_evex_instruction();
10759 attributes.set_embedded_opmask_register_specifier(mask);
10760 if (merge) {
10761 attributes.reset_is_clear_context();
10762 }
10763 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10764 emit_int16(0x58, (0xC0 | encode));
10765 }
10766
10767 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10768 InstructionMark im(this);
10769 assert(VM_Version::supports_evex(), "");
10770 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10771 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10772 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10773 attributes.set_is_evex_instruction();
10774 attributes.set_embedded_opmask_register_specifier(mask);
10775 if (merge) {
10776 attributes.reset_is_clear_context();
10777 }
10778 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10779 emit_int8(0x58);
10780 emit_operand(dst, src, 0);
10781 }
10782
10783 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10784 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10785 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10786 attributes.set_is_evex_instruction();
10787 attributes.set_embedded_opmask_register_specifier(mask);
10788 if (merge) {
10789 attributes.reset_is_clear_context();
10790 }
10791 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10792 emit_int16((unsigned char)0xF8, (0xC0 | encode));
10793 }
10794
10795 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10796 InstructionMark im(this);
10797 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10798 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10799 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10800 attributes.set_is_evex_instruction();
10801 attributes.set_embedded_opmask_register_specifier(mask);
10802 if (merge) {
10803 attributes.reset_is_clear_context();
10804 }
10805 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10806 emit_int8((unsigned char)0xF8);
10807 emit_operand(dst, src, 0);
10808 }
10809
10810 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10811 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10812 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10813 attributes.set_is_evex_instruction();
10814 attributes.set_embedded_opmask_register_specifier(mask);
10815 if (merge) {
10816 attributes.reset_is_clear_context();
10817 }
10818 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10819 emit_int16((unsigned char)0xF9, (0xC0 | encode));
10820 }
10821
10822 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10823 InstructionMark im(this);
10824 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10825 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10826 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10827 attributes.set_is_evex_instruction();
10828 attributes.set_embedded_opmask_register_specifier(mask);
10829 if (merge) {
10830 attributes.reset_is_clear_context();
10831 }
10832 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10833 emit_int8((unsigned char)0xF9);
10834 emit_operand(dst, src, 0);
10835 }
10836
10837 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10838 assert(VM_Version::supports_evex(), "");
10839 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10840 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10841 attributes.set_is_evex_instruction();
10842 attributes.set_embedded_opmask_register_specifier(mask);
10843 if (merge) {
10844 attributes.reset_is_clear_context();
10845 }
10846 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10847 emit_int16((unsigned char)0xFA, (0xC0 | encode));
10848 }
10849
10850 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10851 InstructionMark im(this);
10852 assert(VM_Version::supports_evex(), "");
10853 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10854 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10855 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10856 attributes.set_is_evex_instruction();
10857 attributes.set_embedded_opmask_register_specifier(mask);
10858 if (merge) {
10859 attributes.reset_is_clear_context();
10860 }
10861 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10862 emit_int8((unsigned char)0xFA);
10863 emit_operand(dst, src, 0);
10864 }
10865
10866 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10867 assert(VM_Version::supports_evex(), "");
10868 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10869 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10870 attributes.set_is_evex_instruction();
10871 attributes.set_embedded_opmask_register_specifier(mask);
10872 if (merge) {
10873 attributes.reset_is_clear_context();
10874 }
10875 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10876 emit_int16((unsigned char)0xFB, (0xC0 | encode));
10877 }
10878
10879 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10880 InstructionMark im(this);
10881 assert(VM_Version::supports_evex(), "");
10882 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10883 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10884 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10885 attributes.set_is_evex_instruction();
10886 attributes.set_embedded_opmask_register_specifier(mask);
10887 if (merge) {
10888 attributes.reset_is_clear_context();
10889 }
10890 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10891 emit_int8((unsigned char)0xFB);
10892 emit_operand(dst, src, 0);
10893 }
10894
10895 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10896 assert(VM_Version::supports_evex(), "");
10897 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10898 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10899 attributes.set_is_evex_instruction();
10900 attributes.set_embedded_opmask_register_specifier(mask);
10901 if (merge) {
10902 attributes.reset_is_clear_context();
10903 }
10904 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10905 emit_int16(0x5C, (0xC0 | encode));
10906 }
10907
10908 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10909 InstructionMark im(this);
10910 assert(VM_Version::supports_evex(), "");
10911 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10912 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10913 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10914 attributes.set_is_evex_instruction();
10915 attributes.set_embedded_opmask_register_specifier(mask);
10916 if (merge) {
10917 attributes.reset_is_clear_context();
10918 }
10919 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10920 emit_int8(0x5C);
10921 emit_operand(dst, src, 0);
10922 }
10923
10924 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10925 assert(VM_Version::supports_evex(), "");
10926 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10927 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10928 attributes.set_is_evex_instruction();
10929 attributes.set_embedded_opmask_register_specifier(mask);
10930 if (merge) {
10931 attributes.reset_is_clear_context();
10932 }
10933 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10934 emit_int16(0x5C, (0xC0 | encode));
10935 }
10936
10937 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10938 InstructionMark im(this);
10939 assert(VM_Version::supports_evex(), "");
10940 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10941 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
10943 attributes.set_is_evex_instruction();
10944 attributes.set_embedded_opmask_register_specifier(mask);
10945 if (merge) {
10946 attributes.reset_is_clear_context();
10947 }
10948 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10949 emit_int8(0x5C);
10950 emit_operand(dst, src, 0);
10951 }
10952
10953 void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10954 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10955 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10956 attributes.set_is_evex_instruction();
10957 attributes.set_embedded_opmask_register_specifier(mask);
10958 if (merge) {
10959 attributes.reset_is_clear_context();
10960 }
10961 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10962 emit_int16((unsigned char)0xEC, (0xC0 | encode));
10963 }
10964
10965 void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10966 InstructionMark im(this);
10967 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10968 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10969 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10970 attributes.set_is_evex_instruction();
10971 attributes.set_embedded_opmask_register_specifier(mask);
10972 if (merge) {
10973 attributes.reset_is_clear_context();
10974 }
10975 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10976 emit_int8((unsigned char)0xEC);
10977 emit_operand(dst, src, 0);
10978 }
10979
10980 void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10981 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10982 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10983 attributes.set_is_evex_instruction();
10984 attributes.set_embedded_opmask_register_specifier(mask);
10985 if (merge) {
10986 attributes.reset_is_clear_context();
10987 }
10988 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10989 emit_int16((unsigned char)0xED, (0xC0 | encode));
10990 }
10991
10992 void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10993 InstructionMark im(this);
10994 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10995 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
10996 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
10997 attributes.set_is_evex_instruction();
10998 attributes.set_embedded_opmask_register_specifier(mask);
10999 if (merge) {
11000 attributes.reset_is_clear_context();
11001 }
11002 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11003 emit_int8((unsigned char)0xED);
11004 emit_operand(dst, src, 0);
11005 }
11006
11007 void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11008 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11009 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11010 attributes.set_is_evex_instruction();
11011 attributes.set_embedded_opmask_register_specifier(mask);
11012 if (merge) {
11013 attributes.reset_is_clear_context();
11014 }
11015 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11016 emit_int16((unsigned char)0xDC, (0xC0 | encode));
11017 }
11018
11019 void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11020 InstructionMark im(this);
11021 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11022 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11023 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11024 attributes.set_is_evex_instruction();
11025 attributes.set_embedded_opmask_register_specifier(mask);
11026 if (merge) {
11027 attributes.reset_is_clear_context();
11028 }
11029 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11030 emit_int8((unsigned char)0xDC);
11031 emit_operand(dst, src, 0);
11032 }
11033
11034 void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11035 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11036 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11037 attributes.set_is_evex_instruction();
11038 attributes.set_embedded_opmask_register_specifier(mask);
11039 if (merge) {
11040 attributes.reset_is_clear_context();
11041 }
11042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11043 emit_int16((unsigned char)0xDD, (0xC0 | encode));
11044 }
11045
11046 void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11047 InstructionMark im(this);
11048 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11049 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11050 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11051 attributes.set_is_evex_instruction();
11052 attributes.set_embedded_opmask_register_specifier(mask);
11053 if (merge) {
11054 attributes.reset_is_clear_context();
11055 }
11056 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11057 emit_int8((unsigned char)0xDD);
11058 emit_operand(dst, src, 0);
11059 }
11060
11061 void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11062 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11063 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11064 attributes.set_is_evex_instruction();
11065 attributes.set_embedded_opmask_register_specifier(mask);
11066 if (merge) {
11067 attributes.reset_is_clear_context();
11068 }
11069 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11070 emit_int16((unsigned char)0xE8, (0xC0 | encode));
11071 }
11072
11073 void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11074 InstructionMark im(this);
11075 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11076 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11077 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11078 attributes.set_is_evex_instruction();
11079 attributes.set_embedded_opmask_register_specifier(mask);
11080 if (merge) {
11081 attributes.reset_is_clear_context();
11082 }
11083 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11084 emit_int8((unsigned char)0xE8);
11085 emit_operand(dst, src, 0);
11086 }
11087
11088 void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11089 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11090 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11091 attributes.set_is_evex_instruction();
11092 attributes.set_embedded_opmask_register_specifier(mask);
11093 if (merge) {
11094 attributes.reset_is_clear_context();
11095 }
11096 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11097 emit_int16((unsigned char)0xE9, (0xC0 | encode));
11098 }
11099
11100 void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11101 InstructionMark im(this);
11102 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11103 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11104 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11105 attributes.set_is_evex_instruction();
11106 attributes.set_embedded_opmask_register_specifier(mask);
11107 if (merge) {
11108 attributes.reset_is_clear_context();
11109 }
11110 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11111 emit_int8((unsigned char)0xE9);
11112 emit_operand(dst, src, 0);
11113 }
11114
11115 void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11116 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11118 attributes.set_is_evex_instruction();
11119 attributes.set_embedded_opmask_register_specifier(mask);
11120 if (merge) {
11121 attributes.reset_is_clear_context();
11122 }
11123 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11124 emit_int16((unsigned char)0xD8, (0xC0 | encode));
11125 }
11126
11127 void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11128 InstructionMark im(this);
11129 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11130 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11131 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11132 attributes.set_is_evex_instruction();
11133 attributes.set_embedded_opmask_register_specifier(mask);
11134 if (merge) {
11135 attributes.reset_is_clear_context();
11136 }
11137 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11138 emit_int8((unsigned char)0xD8);
11139 emit_operand(dst, src, 0);
11140 }
11141
11142 void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11143 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11144 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11145 attributes.set_is_evex_instruction();
11146 attributes.set_embedded_opmask_register_specifier(mask);
11147 if (merge) {
11148 attributes.reset_is_clear_context();
11149 }
11150 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11151 emit_int16((unsigned char)0xD9, (0xC0 | encode));
11152 }
11153
11154
11155 void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11156 InstructionMark im(this);
11157 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11158 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11159 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11160 attributes.set_is_evex_instruction();
11161 attributes.set_embedded_opmask_register_specifier(mask);
11162 if (merge) {
11163 attributes.reset_is_clear_context();
11164 }
11165 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11166 emit_int8((unsigned char)0xD9);
11167 emit_operand(dst, src, 0);
11168 }
11169
11170 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11171 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11172 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11173 attributes.set_is_evex_instruction();
11174 attributes.set_embedded_opmask_register_specifier(mask);
11175 if (merge) {
11176 attributes.reset_is_clear_context();
11177 }
11178 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11179 emit_int16((unsigned char)0xD5, (0xC0 | encode));
11180 }
11181
11182 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11183 InstructionMark im(this);
11184 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11185 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11186 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11187 attributes.set_is_evex_instruction();
11188 attributes.set_embedded_opmask_register_specifier(mask);
11189 if (merge) {
11190 attributes.reset_is_clear_context();
11191 }
11192 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11193 emit_int8((unsigned char)0xD5);
11194 emit_operand(dst, src, 0);
11195 }
11196
11197 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11198 assert(VM_Version::supports_evex(), "");
11199 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11200 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11201 attributes.set_is_evex_instruction();
11202 attributes.set_embedded_opmask_register_specifier(mask);
11203 if (merge) {
11204 attributes.reset_is_clear_context();
11205 }
11206 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11207 emit_int16(0x40, (0xC0 | encode));
11208 }
11209
11210 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11211 InstructionMark im(this);
11212 assert(VM_Version::supports_evex(), "");
11213 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11214 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11215 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11216 attributes.set_is_evex_instruction();
11217 attributes.set_embedded_opmask_register_specifier(mask);
11218 if (merge) {
11219 attributes.reset_is_clear_context();
11220 }
11221 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11222 emit_int8(0x40);
11223 emit_operand(dst, src, 0);
11224 }
11225
11226 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11227 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11228 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11229 attributes.set_is_evex_instruction();
11230 attributes.set_embedded_opmask_register_specifier(mask);
11231 if (merge) {
11232 attributes.reset_is_clear_context();
11233 }
11234 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11235 emit_int16(0x40, (0xC0 | encode));
11236 }
11237
11238 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11239 InstructionMark im(this);
11240 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11241 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11242 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11243 attributes.set_is_evex_instruction();
11244 attributes.set_embedded_opmask_register_specifier(mask);
11245 if (merge) {
11246 attributes.reset_is_clear_context();
11247 }
11248 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11249 emit_int8(0x40);
11250 emit_operand(dst, src, 0);
11251 }
11252
11253 void Assembler::evpmulhw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11254 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11255 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11256 attributes.set_is_evex_instruction();
11257 attributes.set_embedded_opmask_register_specifier(mask);
11258 if (merge) {
11259 attributes.reset_is_clear_context();
11260 }
11261 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11262 emit_int16((unsigned char)0xE5, (0xC0 | encode));
11263 }
11264
11265 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11266 assert(VM_Version::supports_evex(), "");
11267 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11268 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11269 attributes.set_is_evex_instruction();
11270 attributes.set_embedded_opmask_register_specifier(mask);
11271 if (merge) {
11272 attributes.reset_is_clear_context();
11273 }
11274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11275 emit_int16(0x59, (0xC0 | encode));
11276 }
11277
11278 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11279 InstructionMark im(this);
11280 assert(VM_Version::supports_evex(), "");
11281 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11282 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11283 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11284 attributes.set_is_evex_instruction();
11285 attributes.set_embedded_opmask_register_specifier(mask);
11286 if (merge) {
11287 attributes.reset_is_clear_context();
11288 }
11289 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11290 emit_int8(0x59);
11291 emit_operand(dst, src, 0);
11292 }
11293
11294 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11295 assert(VM_Version::supports_evex(), "");
11296 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11297 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11298 attributes.set_is_evex_instruction();
11299 attributes.set_embedded_opmask_register_specifier(mask);
11300 if (merge) {
11301 attributes.reset_is_clear_context();
11302 }
11303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11304 emit_int16(0x59, (0xC0 | encode));
11305 }
11306
11307 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11308 InstructionMark im(this);
11309 assert(VM_Version::supports_evex(), "");
11310 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11311 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11312 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11313 attributes.set_is_evex_instruction();
11314 attributes.set_embedded_opmask_register_specifier(mask);
11315 if (merge) {
11316 attributes.reset_is_clear_context();
11317 }
11318 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11319 emit_int8(0x59);
11320 emit_operand(dst, src, 0);
11321 }
11322
11323 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11324 assert(VM_Version::supports_evex(), "");
11325 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11326 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11327 attributes.set_is_evex_instruction();
11328 attributes.set_embedded_opmask_register_specifier(mask);
11329 if (merge) {
11330 attributes.reset_is_clear_context();
11331 }
11332 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11333 emit_int16(0x51, (0xC0 | encode));
11334 }
11335
11336 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11337 InstructionMark im(this);
11338 assert(VM_Version::supports_evex(), "");
11339 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11340 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11341 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11342 attributes.set_is_evex_instruction();
11343 attributes.set_embedded_opmask_register_specifier(mask);
11344 if (merge) {
11345 attributes.reset_is_clear_context();
11346 }
11347 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11348 emit_int8(0x51);
11349 emit_operand(dst, src, 0);
11350 }
11351
11352 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11353 assert(VM_Version::supports_evex(), "");
11354 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11355 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11356 attributes.set_is_evex_instruction();
11357 attributes.set_embedded_opmask_register_specifier(mask);
11358 if (merge) {
11359 attributes.reset_is_clear_context();
11360 }
11361 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11362 emit_int16(0x51, (0xC0 | encode));
11363 }
11364
11365 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11366 InstructionMark im(this);
11367 assert(VM_Version::supports_evex(), "");
11368 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11369 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11370 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11371 attributes.set_is_evex_instruction();
11372 attributes.set_embedded_opmask_register_specifier(mask);
11373 if (merge) {
11374 attributes.reset_is_clear_context();
11375 }
11376 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11377 emit_int8(0x51);
11378 emit_operand(dst, src, 0);
11379 }
11380
11381
11382 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11383 assert(VM_Version::supports_evex(), "");
11384 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11385 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11386 attributes.set_is_evex_instruction();
11387 attributes.set_embedded_opmask_register_specifier(mask);
11388 if (merge) {
11389 attributes.reset_is_clear_context();
11390 }
11391 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11392 emit_int16(0x5E, (0xC0 | encode));
11393 }
11394
11395 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11396 InstructionMark im(this);
11397 assert(VM_Version::supports_evex(), "");
11398 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11399 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11400 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11401 attributes.set_is_evex_instruction();
11402 attributes.set_embedded_opmask_register_specifier(mask);
11403 if (merge) {
11404 attributes.reset_is_clear_context();
11405 }
11406 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11407 emit_int8(0x5E);
11408 emit_operand(dst, src, 0);
11409 }
11410
11411 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11412 assert(VM_Version::supports_evex(), "");
11413 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11414 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11415 attributes.set_is_evex_instruction();
11416 attributes.set_embedded_opmask_register_specifier(mask);
11417 if (merge) {
11418 attributes.reset_is_clear_context();
11419 }
11420 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11421 emit_int16(0x5E, (0xC0 | encode));
11422 }
11423
11424 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11425 InstructionMark im(this);
11426 assert(VM_Version::supports_evex(), "");
11427 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11428 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11429 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11430 attributes.set_is_evex_instruction();
11431 attributes.set_embedded_opmask_register_specifier(mask);
11432 if (merge) {
11433 attributes.reset_is_clear_context();
11434 }
11435 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11436 emit_int8(0x5E);
11437 emit_operand(dst, src, 0);
11438 }
11439
11440 void Assembler::evdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src, EvexRoundPrefix rmode) {
11441 assert(VM_Version::supports_evex(), "");
11442 InstructionAttr attributes(rmode, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11443 attributes.set_extended_context();
11444 attributes.set_is_evex_instruction();
11445 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11446 emit_int16(0x5E, (0xC0 | encode));
11447 }
11448
11449 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11450 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11451 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11452 attributes.set_is_evex_instruction();
11453 attributes.set_embedded_opmask_register_specifier(mask);
11454 if (merge) {
11455 attributes.reset_is_clear_context();
11456 }
11457 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11458 emit_int16(0x1C, (0xC0 | encode));
11459 }
11460
11461
11462 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
11463 InstructionMark im(this);
11464 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11465 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11466 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11467 attributes.set_is_evex_instruction();
11468 attributes.set_embedded_opmask_register_specifier(mask);
11469 if (merge) {
11470 attributes.reset_is_clear_context();
11471 }
11472 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11473 emit_int8(0x1C);
11474 emit_operand(dst, src, 0);
11475 }
11476
11477 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11478 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11479 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11480 attributes.set_is_evex_instruction();
11481 attributes.set_embedded_opmask_register_specifier(mask);
11482 if (merge) {
11483 attributes.reset_is_clear_context();
11484 }
11485 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11486 emit_int16(0x1D, (0xC0 | encode));
11487 }
11488
11489
11490 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
11491 InstructionMark im(this);
11492 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11493 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11494 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM,/* input_size_in_bits */ EVEX_NObit);
11495 attributes.set_is_evex_instruction();
11496 attributes.set_embedded_opmask_register_specifier(mask);
11497 if (merge) {
11498 attributes.reset_is_clear_context();
11499 }
11500 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11501 emit_int8(0x1D);
11502 emit_operand(dst, src, 0);
11503 }
11504
11505 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11506 assert(VM_Version::supports_evex(), "");
11507 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11508 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11509 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11510 attributes.set_is_evex_instruction();
11511 attributes.set_embedded_opmask_register_specifier(mask);
11512 if (merge) {
11513 attributes.reset_is_clear_context();
11514 }
11515 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11516 emit_int16(0x1E, (0xC0 | encode));
11517 }
11518
11519
11520 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
11521 InstructionMark im(this);
11522 assert(VM_Version::supports_evex(), "");
11523 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11524 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11525 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11526 attributes.set_is_evex_instruction();
11527 attributes.set_embedded_opmask_register_specifier(mask);
11528 if (merge) {
11529 attributes.reset_is_clear_context();
11530 }
11531 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11532 emit_int8(0x1E);
11533 emit_operand(dst, src, 0);
11534 }
11535
11536 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11537 assert(VM_Version::supports_evex(), "");
11538 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11539 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11540 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11541 attributes.set_is_evex_instruction();
11542 attributes.set_embedded_opmask_register_specifier(mask);
11543 if (merge) {
11544 attributes.reset_is_clear_context();
11545 }
11546 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11547 emit_int16(0x1F, (0xC0 | encode));
11548 }
11549
11550
11551 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
11552 InstructionMark im(this);
11553 assert(VM_Version::supports_evex(), "");
11554 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11555 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11556 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11557 attributes.set_is_evex_instruction();
11558 attributes.set_embedded_opmask_register_specifier(mask);
11559 if (merge) {
11560 attributes.reset_is_clear_context();
11561 }
11562 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11563 emit_int8(0x1F);
11564 emit_operand(dst, src, 0);
11565 }
11566
11567 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11568 assert(VM_Version::supports_evex(), "");
11569 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11570 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11571 attributes.set_is_evex_instruction();
11572 attributes.set_embedded_opmask_register_specifier(mask);
11573 if (merge) {
11574 attributes.reset_is_clear_context();
11575 }
11576 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11577 emit_int16((unsigned char)0xA8, (0xC0 | encode));
11578 }
11579
11580 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11581 InstructionMark im(this);
11582 assert(VM_Version::supports_evex(), "");
11583 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11584 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11585 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11586 attributes.set_is_evex_instruction();
11587 attributes.set_embedded_opmask_register_specifier(mask);
11588 if (merge) {
11589 attributes.reset_is_clear_context();
11590 }
11591 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11592 emit_int8((unsigned char)0xA8);
11593 emit_operand(dst, src, 0);
11594 }
11595
11596 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11597 assert(VM_Version::supports_evex(), "");
11598 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11599 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11600 attributes.set_is_evex_instruction();
11601 attributes.set_embedded_opmask_register_specifier(mask);
11602 if (merge) {
11603 attributes.reset_is_clear_context();
11604 }
11605 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11606 emit_int16((unsigned char)0xA8, (0xC0 | encode));
11607 }
11608
11609 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11610 InstructionMark im(this);
11611 assert(VM_Version::supports_evex(), "");
11612 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11613 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
11614 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
11615 attributes.set_is_evex_instruction();
11616 attributes.set_embedded_opmask_register_specifier(mask);
11617 if (merge) {
11618 attributes.reset_is_clear_context();
11619 }
11620 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11621 emit_int8((unsigned char)0xA8);
11622 emit_operand(dst, src, 0);
11623 }
11624
11625 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11626 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11627 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11628 attributes.set_is_evex_instruction();
11629 attributes.set_embedded_opmask_register_specifier(mask);
11630 if (merge) {
11631 attributes.reset_is_clear_context();
11632 }
11633 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11634 emit_int16((unsigned char)0x8D, (0xC0 | encode));
11635 }
11636
11637 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11638 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11639 InstructionMark im(this);
11640 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11641 attributes.set_is_evex_instruction();
11642 attributes.set_embedded_opmask_register_specifier(mask);
11643 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11644 if (merge) {
11645 attributes.reset_is_clear_context();
11646 }
11647 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11648 emit_int8((unsigned char)0x8D);
11649 emit_operand(dst, src, 0);
11650 }
11651
11652 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11653 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11654 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11655 attributes.set_is_evex_instruction();
11656 attributes.set_embedded_opmask_register_specifier(mask);
11657 if (merge) {
11658 attributes.reset_is_clear_context();
11659 }
11660 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11661 emit_int16((unsigned char)0x8D, (0xC0 | encode));
11662 }
11663
11664 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11665 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11666 InstructionMark im(this);
11667 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11668 attributes.set_is_evex_instruction();
11669 attributes.set_embedded_opmask_register_specifier(mask);
11670 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11671 if (merge) {
11672 attributes.reset_is_clear_context();
11673 }
11674 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11675 emit_int8((unsigned char)0x8D);
11676 emit_operand(dst, src, 0);
11677 }
11678
11679 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11680 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
11681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11682 attributes.set_is_evex_instruction();
11683 attributes.set_embedded_opmask_register_specifier(mask);
11684 if (merge) {
11685 attributes.reset_is_clear_context();
11686 }
11687 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11688 emit_int16(0x36, (0xC0 | encode));
11689 }
11690
11691 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11692 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
11693 InstructionMark im(this);
11694 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11695 attributes.set_is_evex_instruction();
11696 attributes.set_embedded_opmask_register_specifier(mask);
11697 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11698 if (merge) {
11699 attributes.reset_is_clear_context();
11700 }
11701 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11702 emit_int8(0x36);
11703 emit_operand(dst, src, 0);
11704 }
11705
11706 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11707 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
11708 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11709 attributes.set_is_evex_instruction();
11710 attributes.set_embedded_opmask_register_specifier(mask);
11711 if (merge) {
11712 attributes.reset_is_clear_context();
11713 }
11714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11715 emit_int16(0x36, (0xC0 | encode));
11716 }
11717
11718 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
11719 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
11720 InstructionMark im(this);
11721 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11722 attributes.set_is_evex_instruction();
11723 attributes.set_embedded_opmask_register_specifier(mask);
11724 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11725 if (merge) {
11726 attributes.reset_is_clear_context();
11727 }
11728 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11729 emit_int8(0x36);
11730 emit_operand(dst, src, 0);
11731 }
11732
11733 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11734 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11735 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11736 attributes.set_is_evex_instruction();
11737 attributes.set_embedded_opmask_register_specifier(mask);
11738 if (merge) {
11739 attributes.reset_is_clear_context();
11740 }
11741 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11742 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
11743 }
11744
11745 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11746 assert(VM_Version::supports_evex(), "");
11747 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11748 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11749 attributes.set_is_evex_instruction();
11750 attributes.set_embedded_opmask_register_specifier(mask);
11751 if (merge) {
11752 attributes.reset_is_clear_context();
11753 }
11754 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11755 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11756 }
11757
11758 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11759 assert(VM_Version::supports_evex(), "");
11760 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11761 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11762 attributes.set_is_evex_instruction();
11763 attributes.set_embedded_opmask_register_specifier(mask);
11764 if (merge) {
11765 attributes.reset_is_clear_context();
11766 }
11767 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11768 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
11769 }
11770
11771 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11772 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11773 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11774 attributes.set_is_evex_instruction();
11775 attributes.set_embedded_opmask_register_specifier(mask);
11776 if (merge) {
11777 attributes.reset_is_clear_context();
11778 }
11779 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11780 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
11781 }
11782
11783 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11784 assert(VM_Version::supports_evex(), "");
11785 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11786 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11787 attributes.set_is_evex_instruction();
11788 attributes.set_embedded_opmask_register_specifier(mask);
11789 if (merge) {
11790 attributes.reset_is_clear_context();
11791 }
11792 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11793 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11794 }
11795
11796 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11797 assert(VM_Version::supports_evex(), "");
11798 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11799 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11800 attributes.set_is_evex_instruction();
11801 attributes.set_embedded_opmask_register_specifier(mask);
11802 if (merge) {
11803 attributes.reset_is_clear_context();
11804 }
11805 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11806 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
11807 }
11808
11809 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11810 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11811 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11812 attributes.set_is_evex_instruction();
11813 attributes.set_embedded_opmask_register_specifier(mask);
11814 if (merge) {
11815 attributes.reset_is_clear_context();
11816 }
11817 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11818 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
11819 }
11820
11821 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11822 assert(VM_Version::supports_evex(), "");
11823 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11824 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11825 attributes.set_is_evex_instruction();
11826 attributes.set_embedded_opmask_register_specifier(mask);
11827 if (merge) {
11828 attributes.reset_is_clear_context();
11829 }
11830 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11831 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11832 }
11833
11834 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11835 assert(VM_Version::supports_evex(), "");
11836 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11837 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11838 attributes.set_is_evex_instruction();
11839 attributes.set_embedded_opmask_register_specifier(mask);
11840 if (merge) {
11841 attributes.reset_is_clear_context();
11842 }
11843 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11844 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11845 }
11846
11847 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11848 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11849 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11850 attributes.set_is_evex_instruction();
11851 attributes.set_embedded_opmask_register_specifier(mask);
11852 if (merge) {
11853 attributes.reset_is_clear_context();
11854 }
11855 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11856 emit_int16((unsigned char)0xF1, (0xC0 | encode));
11857 }
11858
11859 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11860 assert(VM_Version::supports_evex(), "");
11861 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11862 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11863 attributes.set_is_evex_instruction();
11864 attributes.set_embedded_opmask_register_specifier(mask);
11865 if (merge) {
11866 attributes.reset_is_clear_context();
11867 }
11868 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11869 emit_int16((unsigned char)0xF2, (0xC0 | encode));
11870 }
11871
11872 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11873 assert(VM_Version::supports_evex(), "");
11874 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11875 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11876 attributes.set_is_evex_instruction();
11877 attributes.set_embedded_opmask_register_specifier(mask);
11878 if (merge) {
11879 attributes.reset_is_clear_context();
11880 }
11881 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11882 emit_int16((unsigned char)0xF3, (0xC0 | encode));
11883 }
11884
11885 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11886 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11887 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11888 attributes.set_is_evex_instruction();
11889 attributes.set_embedded_opmask_register_specifier(mask);
11890 if (merge) {
11891 attributes.reset_is_clear_context();
11892 }
11893 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11894 emit_int16((unsigned char)0xD1, (0xC0 | encode));
11895 }
11896
11897 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11898 assert(VM_Version::supports_evex(), "");
11899 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11900 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11901 attributes.set_is_evex_instruction();
11902 attributes.set_embedded_opmask_register_specifier(mask);
11903 if (merge) {
11904 attributes.reset_is_clear_context();
11905 }
11906 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11907 emit_int16((unsigned char)0xD2, (0xC0 | encode));
11908 }
11909
11910 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11911 assert(VM_Version::supports_evex(), "");
11912 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11913 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11914 attributes.set_is_evex_instruction();
11915 attributes.set_embedded_opmask_register_specifier(mask);
11916 if (merge) {
11917 attributes.reset_is_clear_context();
11918 }
11919 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11920 emit_int16((unsigned char)0xD3, (0xC0 | encode));
11921 }
11922
11923 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11924 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11925 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11926 attributes.set_is_evex_instruction();
11927 attributes.set_embedded_opmask_register_specifier(mask);
11928 if (merge) {
11929 attributes.reset_is_clear_context();
11930 }
11931 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11932 emit_int16((unsigned char)0xE1, (0xC0 | encode));
11933 }
11934
11935 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11936 assert(VM_Version::supports_evex(), "");
11937 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11938 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11939 attributes.set_is_evex_instruction();
11940 attributes.set_embedded_opmask_register_specifier(mask);
11941 if (merge) {
11942 attributes.reset_is_clear_context();
11943 }
11944 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11945 emit_int16((unsigned char)0xE2, (0xC0 | encode));
11946 }
11947
11948 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11949 assert(VM_Version::supports_evex(), "");
11950 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11951 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11952 attributes.set_is_evex_instruction();
11953 attributes.set_embedded_opmask_register_specifier(mask);
11954 if (merge) {
11955 attributes.reset_is_clear_context();
11956 }
11957 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11958 emit_int16((unsigned char)0xE2, (0xC0 | encode));
11959 }
11960
11961 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11962 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
11963 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11964 attributes.set_is_evex_instruction();
11965 attributes.set_embedded_opmask_register_specifier(mask);
11966 if (merge) {
11967 attributes.reset_is_clear_context();
11968 }
11969 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11970 emit_int16(0x12, (0xC0 | encode));
11971 }
11972
11973 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11974 assert(VM_Version::supports_evex(), "");
11975 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11976 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11977 attributes.set_is_evex_instruction();
11978 attributes.set_embedded_opmask_register_specifier(mask);
11979 if (merge) {
11980 attributes.reset_is_clear_context();
11981 }
11982 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11983 emit_int16(0x47, (0xC0 | encode));
11984 }
11985
11986 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11987 assert(VM_Version::supports_evex(), "");
11988 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11989 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11990 attributes.set_is_evex_instruction();
11991 attributes.set_embedded_opmask_register_specifier(mask);
11992 if (merge) {
11993 attributes.reset_is_clear_context();
11994 }
11995 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11996 emit_int16(0x47, (0xC0 | encode));
11997 }
11998
11999 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12000 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12001 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12002 attributes.set_is_evex_instruction();
12003 attributes.set_embedded_opmask_register_specifier(mask);
12004 if (merge) {
12005 attributes.reset_is_clear_context();
12006 }
12007 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12008 emit_int16(0x10, (0xC0 | encode));
12009 }
12010
12011 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12012 assert(VM_Version::supports_evex(), "");
12013 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12014 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12015 attributes.set_is_evex_instruction();
12016 attributes.set_embedded_opmask_register_specifier(mask);
12017 if (merge) {
12018 attributes.reset_is_clear_context();
12019 }
12020 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12021 emit_int16(0x45, (0xC0 | encode));
12022 }
12023
12024 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12025 assert(VM_Version::supports_evex(), "");
12026 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12027 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12028 attributes.set_is_evex_instruction();
12029 attributes.set_embedded_opmask_register_specifier(mask);
12030 if (merge) {
12031 attributes.reset_is_clear_context();
12032 }
12033 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12034 emit_int16(0x45, (0xC0 | encode));
12035 }
12036
12037 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12038 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12039 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12040 attributes.set_is_evex_instruction();
12041 attributes.set_embedded_opmask_register_specifier(mask);
12042 if (merge) {
12043 attributes.reset_is_clear_context();
12044 }
12045 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12046 emit_int16(0x11, (0xC0 | encode));
12047 }
12048
12049 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12050 assert(VM_Version::supports_evex(), "");
12051 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12052 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12053 attributes.set_is_evex_instruction();
12054 attributes.set_embedded_opmask_register_specifier(mask);
12055 if (merge) {
12056 attributes.reset_is_clear_context();
12057 }
12058 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12059 emit_int16(0x46, (0xC0 | encode));
12060 }
12061
12062 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12063 assert(VM_Version::supports_evex(), "");
12064 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12065 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12066 attributes.set_is_evex_instruction();
12067 attributes.set_embedded_opmask_register_specifier(mask);
12068 if (merge) {
12069 attributes.reset_is_clear_context();
12070 }
12071 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12072 emit_int16(0x46, (0xC0 | encode));
12073 }
12074
12075 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12076 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12077 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12078 attributes.set_is_evex_instruction();
12079 attributes.set_embedded_opmask_register_specifier(mask);
12080 if (merge) {
12081 attributes.reset_is_clear_context();
12082 }
12083 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12084 emit_int16(0x38, (0xC0 | encode));
12085 }
12086
12087 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12088 assert(VM_Version::supports_avx512bw(), "");
12089 InstructionMark im(this);
12090 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12091 attributes.set_is_evex_instruction();
12092 attributes.set_embedded_opmask_register_specifier(mask);
12093 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
12094 if (merge) {
12095 attributes.reset_is_clear_context();
12096 }
12097 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12098 emit_int8(0x38);
12099 emit_operand(dst, src, 0);
12100 }
12101
12102 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12103 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12104 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12105 attributes.set_is_evex_instruction();
12106 attributes.set_embedded_opmask_register_specifier(mask);
12107 if (merge) {
12108 attributes.reset_is_clear_context();
12109 }
12110 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12111 emit_int16((unsigned char)0xEA, (0xC0 | encode));
12112 }
12113
12114 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12115 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12116 InstructionMark im(this);
12117 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12118 attributes.set_is_evex_instruction();
12119 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
12120 attributes.set_embedded_opmask_register_specifier(mask);
12121 if (merge) {
12122 attributes.reset_is_clear_context();
12123 }
12124 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12125 emit_int8((unsigned char)0xEA);
12126 emit_operand(dst, src, 0);
12127 }
12128
12129 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12130 assert(VM_Version::supports_evex(), "");
12131 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12132 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12133 attributes.set_is_evex_instruction();
12134 attributes.set_embedded_opmask_register_specifier(mask);
12135 if (merge) {
12136 attributes.reset_is_clear_context();
12137 }
12138 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12139 emit_int16(0x39, (0xC0 | encode));
12140 }
12141
12142 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12143 assert(VM_Version::supports_evex(), "");
12144 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12145 InstructionMark im(this);
12146 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12147 attributes.set_is_evex_instruction();
12148 attributes.set_embedded_opmask_register_specifier(mask);
12149 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12150 if (merge) {
12151 attributes.reset_is_clear_context();
12152 }
12153 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12154 emit_int8(0x39);
12155 emit_operand(dst, src, 0);
12156 }
12157
12158 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12159 assert(VM_Version::supports_evex(), "");
12160 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12161 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12162 attributes.set_is_evex_instruction();
12163 attributes.set_embedded_opmask_register_specifier(mask);
12164 if (merge) {
12165 attributes.reset_is_clear_context();
12166 }
12167 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12168 emit_int16(0x39, (0xC0 | encode));
12169 }
12170
12171 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12172 assert(VM_Version::supports_evex(), "");
12173 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12174 InstructionMark im(this);
12175 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12176 attributes.set_is_evex_instruction();
12177 attributes.set_embedded_opmask_register_specifier(mask);
12178 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12179 if (merge) {
12180 attributes.reset_is_clear_context();
12181 }
12182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12183 emit_int8(0x39);
12184 emit_operand(dst, src, 0);
12185 }
12186
12187
12188 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12189 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12190 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12191 attributes.set_is_evex_instruction();
12192 attributes.set_embedded_opmask_register_specifier(mask);
12193 if (merge) {
12194 attributes.reset_is_clear_context();
12195 }
12196 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12197 emit_int16(0x3C, (0xC0 | encode));
12198 }
12199
12200 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12201 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12202 InstructionMark im(this);
12203 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12204 attributes.set_is_evex_instruction();
12205 attributes.set_embedded_opmask_register_specifier(mask);
12206 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
12207 if (merge) {
12208 attributes.reset_is_clear_context();
12209 }
12210 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12211 emit_int8(0x3C);
12212 emit_operand(dst, src, 0);
12213 }
12214
12215 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12216 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12217 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12218 attributes.set_is_evex_instruction();
12219 attributes.set_embedded_opmask_register_specifier(mask);
12220 if (merge) {
12221 attributes.reset_is_clear_context();
12222 }
12223 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12224 emit_int16((unsigned char)0xEE, (0xC0 | encode));
12225 }
12226
12227 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12228 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
12229 InstructionMark im(this);
12230 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12231 attributes.set_is_evex_instruction();
12232 attributes.set_embedded_opmask_register_specifier(mask);
12233 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
12234 if (merge) {
12235 attributes.reset_is_clear_context();
12236 }
12237 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12238 emit_int8((unsigned char)0xEE);
12239 emit_operand(dst, src, 0);
12240 }
12241
12242 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12243 assert(VM_Version::supports_evex(), "");
12244 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12245 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12246 attributes.set_is_evex_instruction();
12247 attributes.set_embedded_opmask_register_specifier(mask);
12248 if (merge) {
12249 attributes.reset_is_clear_context();
12250 }
12251 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12252 emit_int16(0x3D, (0xC0 | encode));
12253 }
12254
12255 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12256 assert(VM_Version::supports_evex(), "");
12257 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12258 InstructionMark im(this);
12259 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12260 attributes.set_is_evex_instruction();
12261 attributes.set_embedded_opmask_register_specifier(mask);
12262 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12263 if (merge) {
12264 attributes.reset_is_clear_context();
12265 }
12266 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12267 emit_int8(0x3D);
12268 emit_operand(dst, src, 0);
12269 }
12270
12271 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12272 assert(VM_Version::supports_evex(), "");
12273 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12274 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12275 attributes.set_is_evex_instruction();
12276 attributes.set_embedded_opmask_register_specifier(mask);
12277 if (merge) {
12278 attributes.reset_is_clear_context();
12279 }
12280 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12281 emit_int16(0x3D, (0xC0 | encode));
12282 }
12283
12284 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
12285 assert(VM_Version::supports_evex(), "");
12286 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12287 InstructionMark im(this);
12288 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12289 attributes.set_is_evex_instruction();
12290 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12291 attributes.set_embedded_opmask_register_specifier(mask);
12292 if (merge) {
12293 attributes.reset_is_clear_context();
12294 }
12295 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12296 emit_int8(0x3D);
12297 emit_operand(dst, src, 0);
12298 }
12299
12300 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
12301 assert(VM_Version::supports_evex(), "requires EVEX support");
12302 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
12303 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12304 attributes.set_is_evex_instruction();
12305 attributes.set_embedded_opmask_register_specifier(mask);
12306 if (merge) {
12307 attributes.reset_is_clear_context();
12308 }
12309 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12310 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
12311 }
12312
12313 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
12314 assert(VM_Version::supports_evex(), "requires EVEX support");
12315 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
12316 assert(dst != xnoreg, "sanity");
12317 InstructionMark im(this);
12318 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12319 attributes.set_is_evex_instruction();
12320 attributes.set_embedded_opmask_register_specifier(mask);
12321 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12322 if (merge) {
12323 attributes.reset_is_clear_context();
12324 }
12325 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12326 emit_int8(0x25);
12327 emit_operand(dst, src3, 1);
12328 emit_int8(imm8);
12329 }
12330
12331 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
12332 assert(VM_Version::supports_evex(), "requires EVEX support");
12333 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
12334 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12335 attributes.set_is_evex_instruction();
12336 attributes.set_embedded_opmask_register_specifier(mask);
12337 if (merge) {
12338 attributes.reset_is_clear_context();
12339 }
12340 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12341 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
12342 }
12343
12344 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
12345 assert(VM_Version::supports_evex(), "requires EVEX support");
12346 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
12347 assert(dst != xnoreg, "sanity");
12348 InstructionMark im(this);
12349 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12350 attributes.set_is_evex_instruction();
12351 attributes.set_embedded_opmask_register_specifier(mask);
12352 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
12353 if (merge) {
12354 attributes.reset_is_clear_context();
12355 }
12356 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12357 emit_int8(0x25);
12358 emit_operand(dst, src3, 1);
12359 emit_int8(imm8);
12360 }
12361
12362 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
12363 assert(VM_Version::supports_gfni(), "");
12364 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12366 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
12367 }
12368
12369 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
12370 assert(VM_Version::supports_gfni(), "requires GFNI support");
12371 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12372 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12373 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
12374 }
12375
12376 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
12377 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
12378 assert(UseAVX >= 2, "");
12379 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12380 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12381 emit_int16(0x58, (0xC0 | encode));
12382 }
12383
12384 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
12385 assert(VM_Version::supports_avx2(), "");
12386 assert(dst != xnoreg, "sanity");
12387 InstructionMark im(this);
12388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12389 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12390 // swap src<->dst for encoding
12391 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12392 emit_int8(0x58);
12393 emit_operand(dst, src, 0);
12394 }
12395
12396 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
12397 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
12398 assert(VM_Version::supports_avx2(), "");
12399 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12400 attributes.set_rex_vex_w_reverted();
12401 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12402 emit_int16(0x59, (0xC0 | encode));
12403 }
12404
12405 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
12406 assert(VM_Version::supports_avx2(), "");
12407 assert(dst != xnoreg, "sanity");
12408 InstructionMark im(this);
12409 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12410 attributes.set_rex_vex_w_reverted();
12411 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12412 // swap src<->dst for encoding
12413 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12414 emit_int8(0x59);
12415 emit_operand(dst, src, 0);
12416 }
12417
12418 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
12419 assert(vector_len != Assembler::AVX_128bit, "");
12420 assert(VM_Version::supports_evex(), "");
12421 assert(dst != xnoreg, "sanity");
12422 InstructionMark im(this);
12423 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12424 attributes.set_rex_vex_w_reverted();
12425 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
12426 // swap src<->dst for encoding
12427 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12428 emit_int8(0x5A);
12429 emit_operand(dst, src, 0);
12430 }
12431
12432 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
12433 assert(vector_len != Assembler::AVX_128bit, "");
12434 assert(VM_Version::supports_avx512dq(), "");
12435 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12436 attributes.set_rex_vex_w_reverted();
12437 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12438 emit_int16(0x5A, (0xC0 | encode));
12439 }
12440
12441 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
12442 assert(vector_len != Assembler::AVX_128bit, "");
12443 assert(VM_Version::supports_avx512dq(), "");
12444 assert(dst != xnoreg, "sanity");
12445 InstructionMark im(this);
12446 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12447 attributes.set_rex_vex_w_reverted();
12448 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
12449 // swap src<->dst for encoding
12450 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12451 emit_int8(0x5A);
12452 emit_operand(dst, src, 0);
12453 }
12454
12455 void Assembler::vbroadcasti128(XMMRegister dst, Address src, int vector_len) {
12456 assert(VM_Version::supports_avx2(), "");
12457 assert(vector_len == AVX_256bit, "");
12458 assert(dst != xnoreg, "sanity");
12459 InstructionMark im(this);
12460 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12461 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
12462 // swap src<->dst for encoding
12463 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12464 emit_int8(0x5A);
12465 emit_operand(dst, src, 0);
12466 }
12467
12468 // scalar single/double precision replicate
12469
12470 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
12471 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
12472 assert(VM_Version::supports_avx2(), "");
12473 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12474 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12475 emit_int16(0x18, (0xC0 | encode));
12476 }
12477
12478 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
12479 assert(VM_Version::supports_avx(), "");
12480 assert(dst != xnoreg, "sanity");
12481 InstructionMark im(this);
12482 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12483 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12484 // swap src<->dst for encoding
12485 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12486 emit_int8(0x18);
12487 emit_operand(dst, src, 0);
12488 }
12489
12490 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
12491 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
12492 assert(VM_Version::supports_avx2(), "");
12493 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
12494 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12495 attributes.set_rex_vex_w_reverted();
12496 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12497 emit_int16(0x19, (0xC0 | encode));
12498 }
12499
12500 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
12501 assert(VM_Version::supports_avx(), "");
12502 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
12503 assert(dst != xnoreg, "sanity");
12504 InstructionMark im(this);
12505 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12506 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12507 attributes.set_rex_vex_w_reverted();
12508 // swap src<->dst for encoding
12509 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12510 emit_int8(0x19);
12511 emit_operand(dst, src, 0);
12512 }
12513
12514 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
12515 assert(VM_Version::supports_avx(), "");
12516 assert(vector_len == AVX_256bit, "");
12517 assert(dst != xnoreg, "sanity");
12518 InstructionMark im(this);
12519 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12520 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
12521 // swap src<->dst for encoding
12522 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12523 emit_int8(0x1A);
12524 emit_operand(dst, src, 0);
12525 }
12526
12527 void Assembler::evbroadcastf64x2(XMMRegister dst, Address src, int vector_len) {
12528 assert(VM_Version::supports_avx512dq(), "");
12529 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12530 assert(dst != xnoreg, "sanity");
12531 InstructionMark im(this);
12532 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12533 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
12534 attributes.set_is_evex_instruction();
12535 // swap src<->dst for encoding
12536 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12537 emit_int8(0x1A);
12538 emit_operand(dst, src, 0);
12539 }
12540
12541
12542 // gpr source broadcast forms
12543
12544 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
12545 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
12546 assert(VM_Version::supports_avx512bw(), "");
12547 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
12548 attributes.set_is_evex_instruction();
12549 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
12550 emit_int16(0x7A, (0xC0 | encode));
12551 }
12552
12553 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
12554 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
12555 assert(VM_Version::supports_avx512bw(), "");
12556 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
12557 attributes.set_is_evex_instruction();
12558 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes , true);
12559 emit_int16(0x7B, (0xC0 | encode));
12560 }
12561
12562 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
12563 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
12564 assert(VM_Version::supports_evex(), "");
12565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12566 attributes.set_is_evex_instruction();
12567 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
12568 emit_int16(0x7C, (0xC0 | encode));
12569 }
12570
12571 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
12572 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
12573 assert(VM_Version::supports_evex(), "");
12574 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12575 attributes.set_is_evex_instruction();
12576 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
12577 emit_int16(0x7C, (0xC0 | encode));
12578 }
12579
12580 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
12581 assert(VM_Version::supports_avx2(), "");
12582 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand");
12583 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
12584 assert(dst != xnoreg, "sanity");
12585 assert(src.isxmmindex(),"expected to be xmm index");
12586 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12587 InstructionMark im(this);
12588 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12589 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12590 emit_int8((unsigned char)0x90);
12591 emit_operand(dst, src, 0);
12592 }
12593
12594 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
12595 assert(VM_Version::supports_avx2(), "");
12596 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand");
12597 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
12598 assert(dst != xnoreg, "sanity");
12599 assert(src.isxmmindex(),"expected to be xmm index");
12600 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12601 InstructionMark im(this);
12602 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12603 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12604 emit_int8((unsigned char)0x90);
12605 emit_operand(dst, src, 0);
12606 }
12607
12608 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
12609 assert(VM_Version::supports_avx2(), "");
12610 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand");
12611 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
12612 assert(dst != xnoreg, "sanity");
12613 assert(src.isxmmindex(),"expected to be xmm index");
12614 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12615 InstructionMark im(this);
12616 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12617 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12618 emit_int8((unsigned char)0x92);
12619 emit_operand(dst, src, 0);
12620 }
12621
12622 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
12623 assert(VM_Version::supports_avx2(), "");
12624 assert(!needs_eevex(src.base()), "does not support extended gprs as BASE of address operand");
12625 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
12626 assert(dst != xnoreg, "sanity");
12627 assert(src.isxmmindex(),"expected to be xmm index");
12628 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12629 InstructionMark im(this);
12630 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
12631 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12632 emit_int8((unsigned char)0x92);
12633 emit_operand(dst, src, 0);
12634 }
12635 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
12636 assert(VM_Version::supports_evex(), "");
12637 assert(dst != xnoreg, "sanity");
12638 assert(src.isxmmindex(),"expected to be xmm index");
12639 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12640 assert(mask != k0, "instruction will #UD if mask is in k0");
12641 InstructionMark im(this);
12642 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12643 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12644 attributes.reset_is_clear_context();
12645 attributes.set_embedded_opmask_register_specifier(mask);
12646 attributes.set_is_evex_instruction();
12647 // swap src<->dst for encoding
12648 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12649 emit_int8((unsigned char)0x90);
12650 emit_operand(dst, src, 0);
12651 }
12652
12653 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
12654 assert(VM_Version::supports_evex(), "");
12655 assert(dst != xnoreg, "sanity");
12656 assert(src.isxmmindex(),"expected to be xmm index");
12657 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12658 assert(mask != k0, "instruction will #UD if mask is in k0");
12659 InstructionMark im(this);
12660 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12661 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12662 attributes.reset_is_clear_context();
12663 attributes.set_embedded_opmask_register_specifier(mask);
12664 attributes.set_is_evex_instruction();
12665 // swap src<->dst for encoding
12666 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12667 emit_int8((unsigned char)0x90);
12668 emit_operand(dst, src, 0);
12669 }
12670
12671 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
12672 assert(VM_Version::supports_evex(), "");
12673 assert(dst != xnoreg, "sanity");
12674 assert(src.isxmmindex(),"expected to be xmm index");
12675 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12676 assert(mask != k0, "instruction will #UD if mask is in k0");
12677 InstructionMark im(this);
12678 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12679 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12680 attributes.reset_is_clear_context();
12681 attributes.set_embedded_opmask_register_specifier(mask);
12682 attributes.set_is_evex_instruction();
12683 // swap src<->dst for encoding
12684 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12685 emit_int8((unsigned char)0x92);
12686 emit_operand(dst, src, 0);
12687 }
12688
12689 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
12690 assert(VM_Version::supports_evex(), "");
12691 assert(dst != xnoreg, "sanity");
12692 assert(src.isxmmindex(),"expected to be xmm index");
12693 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
12694 assert(mask != k0, "instruction will #UD if mask is in k0");
12695 InstructionMark im(this);
12696 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12697 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12698 attributes.reset_is_clear_context();
12699 attributes.set_embedded_opmask_register_specifier(mask);
12700 attributes.set_is_evex_instruction();
12701 // swap src<->dst for encoding
12702 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12703 emit_int8((unsigned char)0x92);
12704 emit_operand(dst, src, 0);
12705 }
12706
12707 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
12708 assert(VM_Version::supports_evex(), "");
12709 assert(mask != k0, "instruction will #UD if mask is in k0");
12710 InstructionMark im(this);
12711 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12712 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12713 attributes.reset_is_clear_context();
12714 attributes.set_embedded_opmask_register_specifier(mask);
12715 attributes.set_is_evex_instruction();
12716 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12717 emit_int8((unsigned char)0xA0);
12718 emit_operand(src, dst, 0);
12719 }
12720
12721 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
12722 assert(VM_Version::supports_evex(), "");
12723 assert(mask != k0, "instruction will #UD if mask is in k0");
12724 InstructionMark im(this);
12725 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12726 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12727 attributes.reset_is_clear_context();
12728 attributes.set_embedded_opmask_register_specifier(mask);
12729 attributes.set_is_evex_instruction();
12730 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12731 emit_int8((unsigned char)0xA0);
12732 emit_operand(src, dst, 0);
12733 }
12734
12735 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
12736 assert(VM_Version::supports_evex(), "");
12737 assert(mask != k0, "instruction will #UD if mask is in k0");
12738 InstructionMark im(this);
12739 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12740 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
12741 attributes.reset_is_clear_context();
12742 attributes.set_embedded_opmask_register_specifier(mask);
12743 attributes.set_is_evex_instruction();
12744 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12745 emit_int8((unsigned char)0xA2);
12746 emit_operand(src, dst, 0);
12747 }
12748
12749 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
12750 assert(VM_Version::supports_evex(), "");
12751 assert(mask != k0, "instruction will #UD if mask is in k0");
12752 InstructionMark im(this);
12753 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12754 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12755 attributes.reset_is_clear_context();
12756 attributes.set_embedded_opmask_register_specifier(mask);
12757 attributes.set_is_evex_instruction();
12758 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12759 emit_int8((unsigned char)0xA2);
12760 emit_operand(src, dst, 0);
12761 }
12762 // Carry-Less Multiplication Quadword
12763 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
12764 assert(VM_Version::supports_clmul(), "");
12765 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12766 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12767 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
12768 }
12769
12770 // Carry-Less Multiplication Quadword
12771 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
12772 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
12773 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12774 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12775 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
12776 }
12777
12778 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
12779 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
12780 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12781 attributes.set_is_evex_instruction();
12782 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12783 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
12784 }
12785
12786 void Assembler::vzeroupper_uncached() {
12787 if (VM_Version::supports_vzeroupper()) {
12788 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12789 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
12790 emit_int8(0x77);
12791 }
12792 }
12793
12794 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
12795 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
12796 assert(VM_Version::supports_evex(), "");
12797 assert(VM_Version::supports_avx512dq(), "");
12798 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
12799 attributes.set_is_evex_instruction();
12800 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12801 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
12802 }
12803
12804 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
12805 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
12806 assert(VM_Version::supports_evex(), "");
12807 assert(VM_Version::supports_avx512dq(), "");
12808 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
12809 attributes.set_is_evex_instruction();
12810 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
12811 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
12812 }
12813
12814 void Assembler::fld_x(Address adr) {
12815 InstructionMark im(this);
12816 emit_int8((unsigned char)0xDB);
12817 emit_operand32(rbp, adr, 0);
12818 }
12819
12820 void Assembler::fstp_x(Address adr) {
12821 InstructionMark im(this);
12822 emit_int8((unsigned char)0xDB);
12823 emit_operand32(rdi, adr, 0);
12824 }
12825
12826 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
12827 assert(reg->encoding() < 8, "no extended registers");
12828 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
12829 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
12830 }
12831
12832 void Assembler::fld_d(Address adr) {
12833 InstructionMark im(this);
12834 emit_int8((unsigned char)0xDD);
12835 emit_operand32(rax, adr, 0);
12836 }
12837
12838 void Assembler::fprem() {
12839 emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
12840 }
12841
12842 void Assembler::fnstsw_ax() {
12843 emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
12844 }
12845
12846 void Assembler::fstp_d(Address adr) {
12847 InstructionMark im(this);
12848 emit_int8((unsigned char)0xDD);
12849 emit_operand32(rbx, adr, 0);
12850 }
12851
12852 void Assembler::fstp_d(int index) {
12853 emit_farith(0xDD, 0xD8, index);
12854 }
12855
12856 void Assembler::emit_farith(int b1, int b2, int i) {
12857 assert(isByte(b1) && isByte(b2), "wrong opcode");
12858 assert(0 <= i && i < 8, "illegal stack offset");
12859 emit_int16(b1, b2 + i);
12860 }
12861
12862 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
12863 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
12864 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
12865 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
12866
12867 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
12868 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
12869 if (pre > 0) {
12870 emit_int8(simd_pre[pre]);
12871 }
12872 if (rex_w) {
12873 prefixq(adr, xreg);
12874 } else {
12875 prefix(adr, xreg);
12876 }
12877 if (opc > 0) {
12878 emit_int8(0x0F);
12879 int opc2 = simd_opc[opc];
12880 if (opc2 > 0) {
12881 emit_int8(opc2);
12882 }
12883 }
12884 }
12885
12886 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
12887 if (pre > 0) {
12888 emit_int8(simd_pre[pre]);
12889 }
12890 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
12891 if (opc > 0) {
12892 emit_int8(0x0F);
12893 int opc2 = simd_opc[opc];
12894 if (opc2 > 0) {
12895 emit_int8(opc2);
12896 }
12897 }
12898 return encode;
12899 }
12900
12901
12902 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
12903 int vector_len = _attributes->get_vector_len();
12904 bool vex_w = _attributes->is_rex_vex_w();
12905 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
12906 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
12907 byte1 = (~byte1) & 0xE0;
12908 byte1 |= opc;
12909
12910 int byte2 = ((~nds_enc) & 0xf) << 3;
12911 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
12912
12913 emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
12914 } else {
12915 int byte1 = vex_r ? VEX_R : 0;
12916 byte1 = (~byte1) & 0x80;
12917 byte1 |= ((~nds_enc) & 0xf) << 3;
12918 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
12919 emit_int16((unsigned char)VEX_2bytes, byte1);
12920 }
12921 }
12922
12923 // This is a 4 byte encoding
12924 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool eevex_b, bool evex_v,
12925 bool eevex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool no_flags) {
12926 // EVEX 0x62 prefix
12927 // byte1 = EVEX_4bytes;
12928
12929 bool vex_w = _attributes->is_rex_vex_w();
12930 int evex_encoding = (vex_w ? VEX_W : 0);
12931 // EVEX.b is not currently used for broadcast of single element or data rounding modes
12932 _attributes->set_evex_encoding(evex_encoding);
12933
12934 // P0: byte 2, initialized to RXBR'0mmm
12935 // instead of not'd
12936 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
12937 byte2 = (~byte2) & 0xF0;
12938 byte2 |= eevex_b ? EEVEX_B : 0;
12939 // confine opc opcode extensions in mm bits to lower two bits
12940 // of form {0F, 0F_38, 0F_3A, 0F_3C}
12941 byte2 |= opc;
12942
12943 // P1: byte 3 as Wvvvv1pp
12944 int byte3 = ((~nds_enc) & 0xf) << 3;
12945 byte3 |= (eevex_x ? 0 : EEVEX_X);
12946 byte3 |= (vex_w & 1) << 7;
12947 // confine pre opcode extensions in pp bits to lower two bits
12948 // of form {66, F3, F2}
12949 byte3 |= pre;
12950
12951 // P2: byte 4 as zL'Lbv'aaa or 00LXVF00 where V = V4, X(extended context) = ND and F = NF (no flags)
12952 int byte4 = 0;
12953 if (no_flags) {
12954 assert(_attributes->is_no_reg_mask(), "mask register not supported with no_flags");
12955 byte4 |= 0x4;
12956 } else {
12957 // kregs are implemented in the low 3 bits as aaa
12958 byte4 = (_attributes->is_no_reg_mask()) ?
12959 0 :
12960 _attributes->get_embedded_opmask_register_specifier();
12961 }
12962 // EVEX.v` for extending EVEX.vvvv or VIDX
12963 byte4 |= (evex_v ? 0: EVEX_V);
12964 // third EXEC.b for broadcast actions
12965 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
12966 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
12967 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
12968 // last is EVEX.z for zero/merge actions
12969 if (_attributes->is_no_reg_mask() == false &&
12970 _attributes->get_embedded_opmask_register_specifier() != 0) {
12971 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
12972 }
12973 emit_int32(EVEX_4bytes, byte2, byte3, byte4);
12974 }
12975
12976 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool nds_is_ndd, bool no_flags) {
12977 if (adr.base_needs_rex2() || adr.index_needs_rex2() || nds_is_ndd || no_flags) {
12978 assert(UseAPX, "APX features not enabled");
12979 }
12980 if (nds_is_ndd) attributes->set_extended_context();
12981 bool is_extended = adr.base_needs_rex2() || adr.index_needs_rex2() || nds_enc >= 16 || xreg_enc >= 16 || nds_is_ndd;
12982 bool vex_r = (xreg_enc & 8) == 8;
12983 bool vex_b = adr.base_needs_rex();
12984 bool vex_x;
12985 if (adr.isxmmindex()) {
12986 vex_x = adr.xmmindex_needs_rex();
12987 } else {
12988 vex_x = adr.index_needs_rex();
12989 }
12990 set_attributes(attributes);
12991 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
12992 // is allowed in legacy mode and has resources which will fit in it.
12993 // Pure EVEX instructions will have is_evex_instruction set in their definition.
12994 if (!attributes->is_legacy_mode()) {
12995 if (UseAVX > 2 && !attributes->is_evex_instruction()) {
12996 if ((attributes->get_vector_len() != AVX_512bit) && !is_extended) {
12997 attributes->set_is_legacy_mode();
12998 }
12999 }
13000 }
13001
13002 if (UseAVX > 2) {
13003 assert(((!attributes->uses_vl()) ||
13004 (attributes->get_vector_len() == AVX_512bit) ||
13005 (!_legacy_mode_vl) ||
13006 (attributes->is_legacy_mode())),"XMM register should be 0-15");
13007 assert((!is_extended || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
13008 }
13009
13010 if (UseAVX > 2 && !attributes->is_legacy_mode())
13011 {
13012 bool evex_r = (xreg_enc >= 16);
13013 bool evex_v;
13014 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
13015 if (adr.isxmmindex()) {
13016 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
13017 } else {
13018 evex_v = (nds_enc >= 16);
13019 }
13020 bool eevex_x = adr.index_needs_rex2();
13021 bool eevex_b = adr.base_needs_rex2();
13022 attributes->set_is_evex_instruction();
13023 evex_prefix(vex_r, vex_b, vex_x, evex_r, eevex_b, evex_v, eevex_x, nds_enc, pre, opc, no_flags);
13024 } else {
13025 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
13026 attributes->set_rex_vex_w(false);
13027 }
13028 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
13029 }
13030 }
13031
13032 void Assembler::eevex_prefix_ndd(Address adr, int ndd_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) {
13033 attributes->set_is_evex_instruction();
13034 vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ true, no_flags);
13035 }
13036
13037 void Assembler::emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
13038 int size, int opcode_byte, bool no_flags, bool is_map1, bool is_commutative) {
13039 if (is_commutative && is_demotable(no_flags, dst->encoding(), src2->encoding())) {
13040 // Opcode byte adjustment due to mismatch between NDD and equivalent demotable variant
13041 opcode_byte += 2;
13042 if (size == EVEX_64bit) {
13043 emit_prefix_and_int8(get_prefixq(src1, dst, is_map1), opcode_byte);
13044 } else {
13045 // For 32-bit, 16-bit and 8-bit
13046 if (size == EVEX_16bit) {
13047 emit_int8(0x66);
13048 }
13049 prefix(src1, dst, false, is_map1);
13050 emit_int8(opcode_byte);
13051 }
13052 } else {
13053 bool vex_w = (size == EVEX_64bit) ? true : false;
13054 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13055 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
13056 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags);
13057 emit_int8(opcode_byte);
13058 }
13059 emit_operand(src2, src1, 0);
13060 }
13061
13062 void Assembler::emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc,
13063 int size, int opcode_byte, bool no_flags, bool is_map1) {
13064 if (is_demotable(no_flags, dst->encoding(), src1->encoding())) {
13065 if (size == EVEX_64bit) {
13066 emit_prefix_and_int8(get_prefixq(src2, dst, is_map1), opcode_byte);
13067 } else {
13068 // For 32-bit, 16-bit and 8-bit
13069 if (size == EVEX_16bit) {
13070 emit_int8(0x66);
13071 }
13072 prefix(src2, dst, false, is_map1);
13073 emit_int8(opcode_byte);
13074 }
13075 } else {
13076 bool vex_w = (size == EVEX_64bit) ? true : false;
13077 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13078 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
13079 eevex_prefix_ndd(src2, dst->encoding(), src1->encoding(), pre, opc, &attributes, no_flags);
13080 emit_int8(opcode_byte);
13081 }
13082 emit_operand(src1, src2, 0);
13083 }
13084
13085 void Assembler::eevex_prefix_nf(Address adr, int ndd_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) {
13086 attributes->set_is_evex_instruction();
13087 vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ false, no_flags);
13088 }
13089
13090 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool src_is_gpr, bool nds_is_ndd, bool no_flags) {
13091 if (nds_is_ndd || no_flags || (src_is_gpr && src_enc >= 16)) {
13092 assert(UseAPX, "APX features not enabled");
13093 }
13094 if (nds_is_ndd) attributes->set_extended_context();
13095 bool is_extended = dst_enc >= 16 || nds_enc >= 16 || src_enc >=16;
13096 bool vex_r = (dst_enc & 8) == 8;
13097 bool vex_b = (src_enc & 8) == 8;
13098 bool vex_x = false;
13099 set_attributes(attributes);
13100
13101 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
13102 // is allowed in legacy mode and has resources which will fit in it.
13103 // Pure EVEX instructions will have is_evex_instruction set in their definition.
13104 if (!attributes->is_legacy_mode()) {
13105 if (UseAVX > 2 && !attributes->is_evex_instruction()) {
13106 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
13107 !is_extended) {
13108 attributes->set_is_legacy_mode();
13109 }
13110 }
13111 }
13112
13113 if (UseAVX > 2) {
13114 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
13115 // Instruction with uses_vl true are vector instructions
13116 // All the vector instructions with AVX_512bit length can have legacy_mode as false
13117 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
13118 // Rest all should have legacy_mode set as true
13119 assert(((!attributes->uses_vl()) ||
13120 (attributes->get_vector_len() == AVX_512bit) ||
13121 (!_legacy_mode_vl) ||
13122 (attributes->is_legacy_mode())),"XMM register should be 0-15");
13123 // Instruction with legacy_mode true should have dst, nds and src < 15
13124 assert(((!is_extended) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
13125 }
13126
13127 if (UseAVX > 2 && !attributes->is_legacy_mode())
13128 {
13129 bool evex_r = (dst_enc >= 16);
13130 bool evex_v = (nds_enc >= 16);
13131 bool evex_b = (src_enc >= 16) && src_is_gpr;
13132 // can use vex_x as bank extender on rm encoding
13133 vex_x = (src_enc >= 16) && !src_is_gpr;
13134 attributes->set_is_evex_instruction();
13135 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_b, evex_v, false /*eevex_x*/, nds_enc, pre, opc, no_flags);
13136 } else {
13137 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
13138 attributes->set_rex_vex_w(false);
13139 }
13140 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
13141 }
13142
13143 // return modrm byte components for operands
13144 return (((dst_enc & 7) << 3) | (src_enc & 7));
13145 }
13146
13147 void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8_t imm8, VexSimdPrefix pre, VexOpcode opc,
13148 int size, int opcode_byte, bool no_flags, bool is_map1) {
13149 bool is_prefixq = (size == EVEX_64bit) ? true : false;
13150 if (is_demotable(no_flags, dst_enc, nds_enc)) {
13151 int encode = is_prefixq ? prefixq_and_encode(src_enc, dst_enc, is_map1) : prefix_and_encode(src_enc, dst_enc, is_map1);
13152 emit_opcode_prefix_and_encoding((unsigned char)(opcode_byte | 0x80), 0xC0, encode, imm8);
13153 } else {
13154 InstructionAttr attributes(AVX_128bit, is_prefixq, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13155 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
13156 int encode = emit_eevex_prefix_or_demote_ndd(src_enc, dst_enc, nds_enc, pre, opc, &attributes, no_flags);
13157 emit_int24(opcode_byte, (0xC0 | encode), imm8);
13158 }
13159 }
13160
13161 void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
13162 int size, int opcode_byte, bool no_flags, bool is_map1, bool swap, bool is_commutative) {
13163 int encode;
13164 bool is_prefixq = (size == EVEX_64bit) ? true : false;
13165 bool first_operand_demotable = is_demotable(no_flags, dst_enc, nds_enc);
13166 bool second_operand_demotable = is_commutative && is_demotable(no_flags, dst_enc, src_enc);
13167 if (first_operand_demotable || second_operand_demotable) {
13168 if (size == EVEX_16bit) {
13169 emit_int8(0x66);
13170 }
13171 int src = first_operand_demotable ? src_enc : nds_enc;
13172 if (swap) {
13173 encode = is_prefixq ? prefixq_and_encode(dst_enc, src, is_map1) : prefix_and_encode(dst_enc, src, is_map1);
13174 } else {
13175 encode = is_prefixq ? prefixq_and_encode(src, dst_enc, is_map1) : prefix_and_encode(src, dst_enc, is_map1);
13176 }
13177 emit_opcode_prefix_and_encoding((unsigned char)opcode_byte, 0xC0, encode);
13178 } else {
13179 InstructionAttr attributes(AVX_128bit, is_prefixq, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13180 attributes.set_is_evex_instruction();
13181 if (swap) {
13182 encode = vex_prefix_and_encode(nds_enc, dst_enc, src_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13183 } else {
13184 encode = vex_prefix_and_encode(src_enc, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13185 }
13186 emit_int16(opcode_byte, (0xC0 | encode));
13187 }
13188 }
13189
13190 int Assembler::emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
13191 InstructionAttr *attributes, bool no_flags, bool use_prefixq) {
13192 if (is_demotable(no_flags, dst_enc, nds_enc)) {
13193 if (pre == VEX_SIMD_66) {
13194 emit_int8(0x66);
13195 }
13196 return use_prefixq ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
13197 }
13198 attributes->set_is_evex_instruction();
13199 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13200 }
13201
13202 int Assembler::emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, VexSimdPrefix pre, VexOpcode opc,
13203 InstructionAttr *attributes, bool no_flags, bool use_prefixq) {
13204 //Demote RegReg and RegRegImm instructions
13205 if (is_demotable(no_flags, dst_enc, nds_enc)) {
13206 return use_prefixq ? prefixq_and_encode(dst_enc) : prefix_and_encode(dst_enc);
13207 }
13208 attributes->set_is_evex_instruction();
13209 return vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13210 }
13211
13212 int Assembler::emit_eevex_prefix_ndd(int dst_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes, bool no_flags) {
13213 attributes->set_is_evex_instruction();
13214 return vex_prefix_and_encode(0, 0, dst_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13215 }
13216
13217 int Assembler::eevex_prefix_and_encode_nf(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
13218 InstructionAttr *attributes, bool no_flags) {
13219 attributes->set_is_evex_instruction();
13220 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ false, no_flags);
13221 }
13222
13223 void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
13224 int size, int op1, int op2, bool no_flags, bool is_commutative) {
13225 bool demotable = is_demotable(no_flags, dst->encoding(), src1->encoding());
13226 if (!demotable && is_commutative) {
13227 if (is_demotable(no_flags, dst->encoding(), src2->encoding())) {
13228 // swap src1 and src2
13229 Register tmp = src1;
13230 src1 = src2;
13231 src2 = tmp;
13232 }
13233 }
13234 bool vex_w = (size == EVEX_64bit) ? true : false;
13235 bool use_prefixq = vex_w;
13236 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13237 // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
13238 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
13239 (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags, use_prefixq);
13240 emit_arith(op1, op2, src1, src2);
13241 }
13242
13243 void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc,
13244 int size, int op1, int op2, bool no_flags) {
13245 int dst_enc = dst->encoding();
13246 int nds_enc = nds->encoding();
13247 bool demote = is_demotable(no_flags, dst_enc, nds_enc);
13248 if (demote) {
13249 (size == EVEX_64bit) ? (void) prefixq_and_encode(dst_enc) : (void) prefix_and_encode(dst_enc);
13250 } else {
13251 bool vex_w = (size == EVEX_64bit) ? true : false;
13252 InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13253 attributes.set_is_evex_instruction();
13254 vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
13255
13256 }
13257 emit_arith(op1, op2, nds, imm32, demote);
13258 }
13259
13260 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
13261 VexOpcode opc, InstructionAttr *attributes) {
13262 if (UseAVX > 0) {
13263 int xreg_enc = xreg->encoding();
13264 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
13265 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
13266 } else {
13267 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
13268 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
13269 }
13270 }
13271
13272 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
13273 VexOpcode opc, InstructionAttr *attributes, bool src_is_gpr) {
13274 int dst_enc = dst->encoding();
13275 int src_enc = src->encoding();
13276 if (UseAVX > 0) {
13277 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
13278 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, src_is_gpr);
13279 } else {
13280 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
13281 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
13282 }
13283 }
13284
13285 bool Assembler::is_demotable(bool no_flags, int dst_enc, int nds_enc) {
13286 return (!no_flags && dst_enc == nds_enc);
13287 }
13288
13289 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
13290 assert(VM_Version::supports_avx(), "");
13291 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13292 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13293 emit_int16(0x5F, (0xC0 | encode));
13294 }
13295
13296 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
13297 assert(VM_Version::supports_avx(), "");
13298 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13299 attributes.set_rex_vex_w_reverted();
13300 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13301 emit_int16(0x5F, (0xC0 | encode));
13302 }
13303
13304 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
13305 assert(VM_Version::supports_avx(), "");
13306 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13307 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13308 emit_int16(0x5D, (0xC0 | encode));
13309 }
13310
13311 void Assembler::eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
13312 assert(VM_Version::supports_avx10_2(), "");
13313 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13314 attributes.set_is_evex_instruction();
13315 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13316 emit_int24(0x53, (0xC0 | encode), imm8);
13317 }
13318
13319 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
13320 assert(VM_Version::supports_avx(), "");
13321 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13322 attributes.set_rex_vex_w_reverted();
13323 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13324 emit_int16(0x5D, (0xC0 | encode));
13325 }
13326
13327 void Assembler::eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
13328 assert(VM_Version::supports_avx10_2(), "");
13329 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13330 attributes.set_is_evex_instruction();
13331 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13332 emit_int24(0x53, (0xC0 | encode), imm8);
13333 }
13334
13335 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
13336 assert(VM_Version::supports_avx(), "");
13337 assert(vector_len <= AVX_256bit, "");
13338 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
13339 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13340 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
13341 }
13342
13343 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
13344 assert(VM_Version::supports_avx(), "");
13345 assert(vector_len <= AVX_256bit, "");
13346 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
13347 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13348 int src2_enc = src2->encoding();
13349 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
13350 }
13351
13352 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
13353 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
13354 assert(vector_len <= AVX_256bit, "");
13355 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
13356 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13357 int src2_enc = src2->encoding();
13358 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
13359 }
13360
13361 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
13362 assert(VM_Version::supports_avx2(), "");
13363 assert(vector_len <= AVX_256bit, "");
13364 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
13365 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13366 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
13367 }
13368
13369 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
13370 assert(VM_Version::supports_avx(), "");
13371 assert(vector_len <= AVX_256bit, "");
13372 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
13373 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
13374 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
13375 }
13376
13377 void Assembler::evcmpph(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13378 ComparisonPredicateFP comparison, int vector_len) {
13379 assert(VM_Version::supports_avx512_fp16(), "");
13380 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13381 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13382 attributes.set_is_evex_instruction();
13383 attributes.set_embedded_opmask_register_specifier(mask);
13384 attributes.reset_is_clear_context();
13385 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
13386 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
13387 }
13388
13389 void Assembler::evcmpsh(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicateFP comparison) {
13390 assert(VM_Version::supports_avx512_fp16(), "");
13391 InstructionAttr attributes(Assembler::AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13392 attributes.set_is_evex_instruction();
13393 attributes.set_embedded_opmask_register_specifier(mask);
13394 attributes.reset_is_clear_context();
13395 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3A, &attributes);
13396 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
13397 }
13398
13399 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13400 ComparisonPredicateFP comparison, int vector_len) {
13401 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13402 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
13403 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13404 attributes.set_is_evex_instruction();
13405 attributes.set_embedded_opmask_register_specifier(mask);
13406 attributes.reset_is_clear_context();
13407 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
13408 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
13409 }
13410
13411 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13412 ComparisonPredicateFP comparison, int vector_len) {
13413 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13414 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
13415 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13416 attributes.set_is_evex_instruction();
13417 attributes.set_embedded_opmask_register_specifier(mask);
13418 attributes.reset_is_clear_context();
13419 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13420 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
13421 }
13422
13423 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
13424 assert(VM_Version::supports_sse4_1(), "");
13425 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
13426 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13427 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13428 emit_int16(0x14, (0xC0 | encode));
13429 }
13430
13431 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
13432 assert(VM_Version::supports_sse4_1(), "");
13433 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
13434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13435 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13436 emit_int16(0x15, (0xC0 | encode));
13437 }
13438
13439 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
13440 assert(VM_Version::supports_sse4_1(), "");
13441 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
13442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13443 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13444 emit_int16(0x10, (0xC0 | encode));
13445 }
13446
13447 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
13448 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
13449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13450 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13451 int src2_enc = src2->encoding();
13452 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
13453 }
13454
13455 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
13456 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13457 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13458 emit_int24(0x0C, (0xC0 | encode), imm8);
13459 }
13460
13461 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
13462 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
13463 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
13464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13465 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13466 emit_int16(0x64, (0xC0 | encode));
13467 }
13468
13469 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
13470 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
13471 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
13472 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13473 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13474 emit_int16(0x65, (0xC0 | encode));
13475 }
13476
13477 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
13478 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
13479 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
13480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13481 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13482 emit_int16(0x66, (0xC0 | encode));
13483 }
13484
13485 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
13486 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
13487 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
13488 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13489 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13490 emit_int16(0x37, (0xC0 | encode));
13491 }
13492
13493 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13494 int comparison, bool is_signed, int vector_len) {
13495 assert(VM_Version::supports_evex(), "");
13496 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13497 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
13498 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13499 attributes.set_is_evex_instruction();
13500 attributes.set_embedded_opmask_register_specifier(mask);
13501 attributes.reset_is_clear_context();
13502 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13503 int opcode = is_signed ? 0x1F : 0x1E;
13504 emit_int24(opcode, (0xC0 | encode), comparison);
13505 }
13506
13507 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
13508 int comparison, bool is_signed, int vector_len) {
13509 assert(VM_Version::supports_evex(), "");
13510 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13511 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
13512 InstructionMark im(this);
13513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13514 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
13515 attributes.set_is_evex_instruction();
13516 attributes.set_embedded_opmask_register_specifier(mask);
13517 attributes.reset_is_clear_context();
13518 int dst_enc = kdst->encoding();
13519 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13520 int opcode = is_signed ? 0x1F : 0x1E;
13521 emit_int8((unsigned char)opcode);
13522 emit_operand(as_Register(dst_enc), src, 1);
13523 emit_int8((unsigned char)comparison);
13524 }
13525
13526 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13527 int comparison, bool is_signed, int vector_len) {
13528 assert(VM_Version::supports_evex(), "");
13529 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13530 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
13531 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13532 attributes.set_is_evex_instruction();
13533 attributes.set_embedded_opmask_register_specifier(mask);
13534 attributes.reset_is_clear_context();
13535 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13536 int opcode = is_signed ? 0x1F : 0x1E;
13537 emit_int24(opcode, (0xC0 | encode), comparison);
13538 }
13539
13540 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
13541 int comparison, bool is_signed, int vector_len) {
13542 assert(VM_Version::supports_evex(), "");
13543 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13544 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
13545 InstructionMark im(this);
13546 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13547 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
13548 attributes.set_is_evex_instruction();
13549 attributes.set_embedded_opmask_register_specifier(mask);
13550 attributes.reset_is_clear_context();
13551 int dst_enc = kdst->encoding();
13552 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13553 int opcode = is_signed ? 0x1F : 0x1E;
13554 emit_int8((unsigned char)opcode);
13555 emit_operand(as_Register(dst_enc), src, 1);
13556 emit_int8((unsigned char)comparison);
13557 }
13558
13559 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13560 int comparison, bool is_signed, int vector_len) {
13561 assert(VM_Version::supports_evex(), "");
13562 assert(VM_Version::supports_avx512bw(), "");
13563 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13564 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
13565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13566 attributes.set_is_evex_instruction();
13567 attributes.set_embedded_opmask_register_specifier(mask);
13568 attributes.reset_is_clear_context();
13569 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13570 int opcode = is_signed ? 0x3F : 0x3E;
13571 emit_int24(opcode, (0xC0 | encode), comparison);
13572 }
13573
13574 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
13575 int comparison, bool is_signed, int vector_len) {
13576 assert(VM_Version::supports_evex(), "");
13577 assert(VM_Version::supports_avx512bw(), "");
13578 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13579 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
13580 InstructionMark im(this);
13581 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13582 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
13583 attributes.set_is_evex_instruction();
13584 attributes.set_embedded_opmask_register_specifier(mask);
13585 attributes.reset_is_clear_context();
13586 int dst_enc = kdst->encoding();
13587 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13588 int opcode = is_signed ? 0x3F : 0x3E;
13589 emit_int8((unsigned char)opcode);
13590 emit_operand(as_Register(dst_enc), src, 1);
13591 emit_int8((unsigned char)comparison);
13592 }
13593
13594 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
13595 int comparison, bool is_signed, int vector_len) {
13596 assert(VM_Version::supports_evex(), "");
13597 assert(VM_Version::supports_avx512bw(), "");
13598 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13599 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
13600 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13601 attributes.set_is_evex_instruction();
13602 attributes.set_embedded_opmask_register_specifier(mask);
13603 attributes.reset_is_clear_context();
13604 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13605 int opcode = is_signed ? 0x3F : 0x3E;
13606 emit_int24(opcode, (0xC0 | encode), comparison);
13607 }
13608
13609 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
13610 int comparison, bool is_signed, int vector_len) {
13611 assert(VM_Version::supports_evex(), "");
13612 assert(VM_Version::supports_avx512bw(), "");
13613 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
13614 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
13615 InstructionMark im(this);
13616 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13617 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
13618 attributes.set_is_evex_instruction();
13619 attributes.set_embedded_opmask_register_specifier(mask);
13620 attributes.reset_is_clear_context();
13621 int dst_enc = kdst->encoding();
13622 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13623 int opcode = is_signed ? 0x3F : 0x3E;
13624 emit_int8((unsigned char)opcode);
13625 emit_operand(as_Register(dst_enc), src, 1);
13626 emit_int8((unsigned char)comparison);
13627 }
13628
13629 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
13630 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13631 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13632 attributes.set_is_evex_instruction();
13633 attributes.set_embedded_opmask_register_specifier(mask);
13634 if (merge) {
13635 attributes.reset_is_clear_context();
13636 }
13637 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13638 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
13639 }
13640
13641 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
13642 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13643 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13644 attributes.set_is_evex_instruction();
13645 attributes.set_embedded_opmask_register_specifier(mask);
13646 if (merge) {
13647 attributes.reset_is_clear_context();
13648 }
13649 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13650 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
13651 }
13652
13653 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13654 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13655 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13656 attributes.set_is_evex_instruction();
13657 attributes.set_embedded_opmask_register_specifier(mask);
13658 if (merge) {
13659 attributes.reset_is_clear_context();
13660 }
13661 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13662 emit_int16(0x14, (0xC0 | encode));
13663 }
13664
13665 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13666 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13667 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13668 attributes.set_is_evex_instruction();
13669 attributes.set_embedded_opmask_register_specifier(mask);
13670 if (merge) {
13671 attributes.reset_is_clear_context();
13672 }
13673 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13674 emit_int16(0x14, (0xC0 | encode));
13675 }
13676
13677 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
13678 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13679 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13680 attributes.set_is_evex_instruction();
13681 attributes.set_embedded_opmask_register_specifier(mask);
13682 if (merge) {
13683 attributes.reset_is_clear_context();
13684 }
13685 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13686 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
13687 }
13688
13689 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
13690 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13691 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13692 attributes.set_is_evex_instruction();
13693 attributes.set_embedded_opmask_register_specifier(mask);
13694 if (merge) {
13695 attributes.reset_is_clear_context();
13696 }
13697 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13698 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
13699 }
13700
13701 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13702 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13703 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13704 attributes.set_is_evex_instruction();
13705 attributes.set_embedded_opmask_register_specifier(mask);
13706 if (merge) {
13707 attributes.reset_is_clear_context();
13708 }
13709 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13710 emit_int16(0x15, (0xC0 | encode));
13711 }
13712
13713 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13714 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
13715 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13716 attributes.set_is_evex_instruction();
13717 attributes.set_embedded_opmask_register_specifier(mask);
13718 if (merge) {
13719 attributes.reset_is_clear_context();
13720 }
13721 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13722 emit_int16(0x15, (0xC0 | encode));
13723 }
13724
13725 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
13726 assert(VM_Version::supports_avx(), "");
13727 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13728 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
13729 int mask_enc = mask->encoding();
13730 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
13731 }
13732
13733 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13734 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13735 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
13736 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13737 attributes.set_is_evex_instruction();
13738 attributes.set_embedded_opmask_register_specifier(mask);
13739 if (merge) {
13740 attributes.reset_is_clear_context();
13741 }
13742 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13743 emit_int16(0x65, (0xC0 | encode));
13744 }
13745
13746 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13747 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13748 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
13749 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13750 attributes.set_is_evex_instruction();
13751 attributes.set_embedded_opmask_register_specifier(mask);
13752 if (merge) {
13753 attributes.reset_is_clear_context();
13754 }
13755 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13756 emit_int16(0x65, (0xC0 | encode));
13757 }
13758
13759 void Assembler::evpblendmb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13760 assert(VM_Version::supports_avx512bw(), "");
13761 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13762 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
13763 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13764 attributes.set_is_evex_instruction();
13765 attributes.set_embedded_opmask_register_specifier(mask);
13766 if (merge) {
13767 attributes.reset_is_clear_context();
13768 }
13769 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13770 emit_int16(0x66, (0xC0 | encode));
13771 }
13772
13773 void Assembler::evpblendmw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13774 assert(VM_Version::supports_avx512bw(), "");
13775 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13776 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
13777 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
13778 attributes.set_is_evex_instruction();
13779 attributes.set_embedded_opmask_register_specifier(mask);
13780 if (merge) {
13781 attributes.reset_is_clear_context();
13782 }
13783 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13784 emit_int16(0x66, (0xC0 | encode));
13785 }
13786
13787 void Assembler::evpblendmd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13788 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13789 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
13790 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13791 attributes.set_is_evex_instruction();
13792 attributes.set_embedded_opmask_register_specifier(mask);
13793 if (merge) {
13794 attributes.reset_is_clear_context();
13795 }
13796 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13797 emit_int16(0x64, (0xC0 | encode));
13798 }
13799
13800 void Assembler::evpblendmq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
13801 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
13802 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
13803 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
13804 attributes.set_is_evex_instruction();
13805 attributes.set_embedded_opmask_register_specifier(mask);
13806 if (merge) {
13807 attributes.reset_is_clear_context();
13808 }
13809 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13810 emit_int16(0x64, (0xC0 | encode));
13811 }
13812
13813 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
13814 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13815 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13816 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
13817 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13818 }
13819
13820 void Assembler::bzhil(Register dst, Register src1, Register src2) {
13821 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13822 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13823 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
13824 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13825 }
13826
13827 void Assembler::pextl(Register dst, Register src1, Register src2) {
13828 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13829 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13830 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
13831 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13832 }
13833
13834 void Assembler::pdepl(Register dst, Register src1, Register src2) {
13835 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13836 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13837 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
13838 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13839 }
13840
13841 void Assembler::pextq(Register dst, Register src1, Register src2) {
13842 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13843 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13844 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
13845 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13846 }
13847
13848 void Assembler::pdepq(Register dst, Register src1, Register src2) {
13849 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13850 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13851 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
13852 emit_int16((unsigned char)0xF5, (0xC0 | encode));
13853 }
13854
13855 void Assembler::pextl(Register dst, Register src1, Address src2) {
13856 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13857 InstructionMark im(this);
13858 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13859 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
13860 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
13861 emit_int8((unsigned char)0xF5);
13862 emit_operand(dst, src2, 0);
13863 }
13864
13865 void Assembler::pdepl(Register dst, Register src1, Address src2) {
13866 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13867 InstructionMark im(this);
13868 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13869 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
13870 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13871 emit_int8((unsigned char)0xF5);
13872 emit_operand(dst, src2, 0);
13873 }
13874
13875 void Assembler::pextq(Register dst, Register src1, Address src2) {
13876 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13877 InstructionMark im(this);
13878 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13879 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
13880 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
13881 emit_int8((unsigned char)0xF5);
13882 emit_operand(dst, src2, 0);
13883 }
13884
13885 void Assembler::pdepq(Register dst, Register src1, Address src2) {
13886 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13887 InstructionMark im(this);
13888 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13889 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
13890 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13891 emit_int8((unsigned char)0xF5);
13892 emit_operand(dst, src2, 0);
13893 }
13894
13895 void Assembler::sarxl(Register dst, Register src1, Register src2) {
13896 assert(VM_Version::supports_bmi2(), "");
13897 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13898 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
13899 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13900 }
13901
13902 void Assembler::sarxl(Register dst, Address src1, Register src2) {
13903 assert(VM_Version::supports_bmi2(), "");
13904 InstructionMark im(this);
13905 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13906 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
13907 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
13908 emit_int8((unsigned char)0xF7);
13909 emit_operand(dst, src1, 0);
13910 }
13911
13912 void Assembler::sarxq(Register dst, Register src1, Register src2) {
13913 assert(VM_Version::supports_bmi2(), "");
13914 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13915 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
13916 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13917 }
13918
13919 void Assembler::sarxq(Register dst, Address src1, Register src2) {
13920 assert(VM_Version::supports_bmi2(), "");
13921 InstructionMark im(this);
13922 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13923 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
13924 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
13925 emit_int8((unsigned char)0xF7);
13926 emit_operand(dst, src1, 0);
13927 }
13928
13929 void Assembler::shlxl(Register dst, Register src1, Register src2) {
13930 assert(VM_Version::supports_bmi2(), "");
13931 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13932 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
13933 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13934 }
13935
13936 void Assembler::shlxl(Register dst, Address src1, Register src2) {
13937 assert(VM_Version::supports_bmi2(), "");
13938 InstructionMark im(this);
13939 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13940 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
13941 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13942 emit_int8((unsigned char)0xF7);
13943 emit_operand(dst, src1, 0);
13944 }
13945
13946 void Assembler::shlxq(Register dst, Register src1, Register src2) {
13947 assert(VM_Version::supports_bmi2(), "");
13948 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13949 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
13950 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13951 }
13952
13953 void Assembler::shlxq(Register dst, Address src1, Register src2) {
13954 assert(VM_Version::supports_bmi2(), "");
13955 InstructionMark im(this);
13956 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13957 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
13958 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
13959 emit_int8((unsigned char)0xF7);
13960 emit_operand(dst, src1, 0);
13961 }
13962
13963 void Assembler::shrxl(Register dst, Register src1, Register src2) {
13964 assert(VM_Version::supports_bmi2(), "");
13965 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13966 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
13967 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13968 }
13969
13970 void Assembler::shrxl(Register dst, Address src1, Register src2) {
13971 assert(VM_Version::supports_bmi2(), "");
13972 InstructionMark im(this);
13973 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13974 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
13975 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13976 emit_int8((unsigned char)0xF7);
13977 emit_operand(dst, src1, 0);
13978 }
13979
13980 void Assembler::shrxq(Register dst, Register src1, Register src2) {
13981 assert(VM_Version::supports_bmi2(), "");
13982 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13983 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
13984 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13985 }
13986
13987 void Assembler::shrxq(Register dst, Address src1, Register src2) {
13988 assert(VM_Version::supports_bmi2(), "");
13989 InstructionMark im(this);
13990 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13991 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
13992 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13993 emit_int8((unsigned char)0xF7);
13994 emit_operand(dst, src1, 0);
13995 }
13996
13997 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
13998 assert(VM_Version::supports_avx512vldq(), "");
13999 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14000 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14001 attributes.set_is_evex_instruction();
14002 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14003 emit_int16(0x39, (0xC0 | encode));
14004 }
14005
14006 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
14007 assert(VM_Version::supports_avx512vldq(), "");
14008 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14009 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14010 attributes.set_is_evex_instruction();
14011 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14012 emit_int16(0x39, (0xC0 | encode));
14013 }
14014
14015 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
14016 assert(VM_Version::supports_avx512vlbw(), "");
14017 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14018 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14019 attributes.set_is_evex_instruction();
14020 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14021 emit_int16(0x29, (0xC0 | encode));
14022 }
14023
14024 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
14025 assert(VM_Version::supports_avx512vlbw(), "");
14026 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14027 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14028 attributes.set_is_evex_instruction();
14029 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14030 emit_int16(0x29, (0xC0 | encode));
14031 }
14032
14033 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
14034 assert(VM_Version::supports_avx512vldq(), "");
14035 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14036 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14037 attributes.set_is_evex_instruction();
14038 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14039 emit_int16(0x38, (0xC0 | encode));
14040 }
14041
14042 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
14043 assert(VM_Version::supports_avx512vldq(), "");
14044 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14045 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14046 attributes.set_is_evex_instruction();
14047 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14048 emit_int16(0x38, (0xC0 | encode));
14049 }
14050
14051 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
14052 assert(VM_Version::supports_avx512vlbw(), "");
14053 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14054 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14055 attributes.set_is_evex_instruction();
14056 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14057 emit_int16(0x28, (0xC0 | encode));
14058 }
14059
14060 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
14061 assert(VM_Version::supports_avx512vlbw(), "");
14062 assert(VM_Version::supports_avx512vl() || vector_len == Assembler::AVX_512bit, "");
14063 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
14064 attributes.set_is_evex_instruction();
14065 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
14066 emit_int16(0x28, (0xC0 | encode));
14067 }
14068
14069 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14070 assert(VM_Version::supports_avx512_vbmi2(), "");
14071 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14072 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14073 attributes.set_embedded_opmask_register_specifier(mask);
14074 attributes.set_is_evex_instruction();
14075 if (merge) {
14076 attributes.reset_is_clear_context();
14077 }
14078 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14079 emit_int16((unsigned char)0x63, (0xC0 | encode));
14080 }
14081
14082 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14083 assert(VM_Version::supports_avx512_vbmi2(), "");
14084 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14085 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14086 attributes.set_embedded_opmask_register_specifier(mask);
14087 attributes.set_is_evex_instruction();
14088 if (merge) {
14089 attributes.reset_is_clear_context();
14090 }
14091 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14092 emit_int16((unsigned char)0x63, (0xC0 | encode));
14093 }
14094
14095 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14096 assert(VM_Version::supports_evex(), "");
14097 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14098 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14099 attributes.set_embedded_opmask_register_specifier(mask);
14100 attributes.set_is_evex_instruction();
14101 if (merge) {
14102 attributes.reset_is_clear_context();
14103 }
14104 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14105 emit_int16((unsigned char)0x8B, (0xC0 | encode));
14106 }
14107
14108 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14109 assert(VM_Version::supports_evex(), "");
14110 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14111 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14112 attributes.set_embedded_opmask_register_specifier(mask);
14113 attributes.set_is_evex_instruction();
14114 if (merge) {
14115 attributes.reset_is_clear_context();
14116 }
14117 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14118 emit_int16((unsigned char)0x8B, (0xC0 | encode));
14119 }
14120
14121 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14122 assert(VM_Version::supports_evex(), "");
14123 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14124 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14125 attributes.set_embedded_opmask_register_specifier(mask);
14126 attributes.set_is_evex_instruction();
14127 if (merge) {
14128 attributes.reset_is_clear_context();
14129 }
14130 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14131 emit_int16((unsigned char)0x8A, (0xC0 | encode));
14132 }
14133
14134 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
14135 assert(VM_Version::supports_evex(), "");
14136 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
14137 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
14138 attributes.set_embedded_opmask_register_specifier(mask);
14139 attributes.set_is_evex_instruction();
14140 if (merge) {
14141 attributes.reset_is_clear_context();
14142 }
14143 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
14144 emit_int16((unsigned char)0x8A, (0xC0 | encode));
14145 }
14146
14147 // This should only be used by 64bit instructions that can use rip-relative
14148 // it cannot be used by instructions that want an immediate value.
14149
14150 // Determine whether an address is always reachable in rip-relative addressing mode
14151 // when accessed from the code cache.
14152 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
14153 switch (reloc_type) {
14154 // This should be rip-relative and easily reachable.
14155 case relocInfo::internal_word_type: {
14156 return true;
14157 }
14158 // This should be rip-relative within the code cache and easily
14159 // reachable until we get huge code caches. (At which point
14160 // IC code is going to have issues).
14161 case relocInfo::virtual_call_type:
14162 case relocInfo::opt_virtual_call_type:
14163 case relocInfo::static_call_type:
14164 case relocInfo::static_stub_type: {
14165 return true;
14166 }
14167 case relocInfo::runtime_call_type:
14168 case relocInfo::external_word_type:
14169 case relocInfo::poll_return_type: // these are really external_word but need special
14170 case relocInfo::poll_type: { // relocs to identify them
14171 return CodeCache::contains(target);
14172 }
14173 default: {
14174 return false;
14175 }
14176 }
14177 }
14178
14179 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
14180 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
14181 if (is_always_reachable(target, reloc_type)) {
14182 return true;
14183 }
14184 switch (reloc_type) {
14185 // None will force a 64bit literal to the code stream. Likely a placeholder
14186 // for something that will be patched later and we need to certain it will
14187 // always be reachable.
14188 case relocInfo::none: {
14189 return false;
14190 }
14191 case relocInfo::runtime_call_type:
14192 case relocInfo::external_word_type:
14193 case relocInfo::poll_return_type: // these are really external_word but need special
14194 case relocInfo::poll_type: { // relocs to identify them
14195 assert(!CodeCache::contains(target), "always reachable");
14196 if (ForceUnreachable) {
14197 return false; // stress the correction code
14198 }
14199 // For external_word_type/runtime_call_type if it is reachable from where we
14200 // are now (possibly a temp buffer) and where we might end up
14201 // anywhere in the code cache then we are always reachable.
14202 // This would have to change if we ever save/restore shared code to be more pessimistic.
14203 // Code buffer has to be allocated in the code cache, so check against
14204 // code cache boundaries cover that case.
14205 //
14206 // In rip-relative addressing mode, an effective address is formed by adding displacement
14207 // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
14208 // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
14209 // to account for that.
14210 return Assembler::is_simm32(target - CodeCache::low_bound()) &&
14211 Assembler::is_simm32(target - CodeCache::high_bound());
14212 }
14213 default: {
14214 return false;
14215 }
14216 }
14217 }
14218
14219 bool Assembler::reachable(AddressLiteral adr) {
14220 assert(CodeCache::contains(pc()), "required");
14221 if (adr.is_lval()) {
14222 return false;
14223 }
14224 return is_reachable(adr.target(), adr.reloc());
14225 }
14226
14227 bool Assembler::always_reachable(AddressLiteral adr) {
14228 assert(CodeCache::contains(pc()), "required");
14229 if (adr.is_lval()) {
14230 return false;
14231 }
14232 return is_always_reachable(adr.target(), adr.reloc());
14233 }
14234
14235 void Assembler::emit_data64(jlong data,
14236 relocInfo::relocType rtype,
14237 int format) {
14238 if (rtype == relocInfo::none) {
14239 emit_int64(data);
14240 } else {
14241 emit_data64(data, Relocation::spec_simple(rtype), format);
14242 }
14243 }
14244
14245 void Assembler::emit_data64(jlong data,
14246 RelocationHolder const& rspec,
14247 int format) {
14248 assert(imm_operand == 0, "default format must be immediate in this file");
14249 assert(imm_operand == format, "must be immediate");
14250 assert(inst_mark() != nullptr, "must be inside InstructionMark");
14251 // Do not use AbstractAssembler::relocate, which is not intended for
14252 // embedded words. Instead, relocate to the enclosing instruction.
14253 code_section()->relocate(inst_mark(), rspec, format);
14254 #ifdef ASSERT
14255 check_relocation(rspec, format);
14256 #endif
14257 emit_int64(data);
14258 }
14259
14260 int Assembler::get_base_prefix_bits(int enc) {
14261 int bits = 0;
14262 if (enc & 16) bits |= REX2BIT_B4;
14263 if (enc & 8) bits |= REX2BIT_B;
14264 return bits;
14265 }
14266
14267 int Assembler::get_index_prefix_bits(int enc) {
14268 int bits = 0;
14269 if (enc & 16) bits |= REX2BIT_X4;
14270 if (enc & 8) bits |= REX2BIT_X;
14271 return bits;
14272 }
14273
14274 int Assembler::get_base_prefix_bits(Register base) {
14275 return base->is_valid() ? get_base_prefix_bits(base->encoding()) : 0;
14276 }
14277
14278 int Assembler::get_index_prefix_bits(Register index) {
14279 return index->is_valid() ? get_index_prefix_bits(index->encoding()) : 0;
14280 }
14281
14282 int Assembler::get_reg_prefix_bits(int enc) {
14283 int bits = 0;
14284 if (enc & 16) bits |= REX2BIT_R4;
14285 if (enc & 8) bits |= REX2BIT_R;
14286 return bits;
14287 }
14288
14289 void Assembler::prefix(Register reg) {
14290 if (reg->encoding() >= 16) {
14291 prefix16(WREX2 | get_base_prefix_bits(reg->encoding()));
14292 } else if (reg->encoding() >= 8) {
14293 prefix(REX_B);
14294 }
14295 }
14296
14297 void Assembler::prefix(Register dst, Register src, Prefix p) {
14298 if ((p & WREX2) || src->encoding() >= 16 || dst->encoding() >= 16) {
14299 prefix_rex2(dst, src);
14300 return;
14301 }
14302 if (src->encoding() >= 8) {
14303 p = (Prefix)(p | REX_B);
14304 }
14305 if (dst->encoding() >= 8) {
14306 p = (Prefix)(p | REX_R);
14307 }
14308 if (p != Prefix_EMPTY) {
14309 // do not generate an empty prefix
14310 prefix(p);
14311 }
14312 }
14313
14314 void Assembler::prefix_rex2(Register dst, Register src) {
14315 int bits = 0;
14316 bits |= get_base_prefix_bits(src->encoding());
14317 bits |= get_reg_prefix_bits(dst->encoding());
14318 prefix16(WREX2 | bits);
14319 }
14320
14321 void Assembler::prefix(Register dst, Address adr, Prefix p) {
14322 if (adr.base_needs_rex2() || adr.index_needs_rex2() || dst->encoding() >= 16) {
14323 prefix_rex2(dst, adr);
14324 }
14325 if (adr.base_needs_rex()) {
14326 if (adr.index_needs_rex()) {
14327 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
14328 } else {
14329 p = (Prefix)(p | REX_B);
14330 }
14331 } else {
14332 if (adr.index_needs_rex()) {
14333 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
14334 }
14335 }
14336 if (dst->encoding() >= 8) {
14337 p = (Prefix)(p | REX_R);
14338 }
14339 if (p != Prefix_EMPTY) {
14340 // do not generate an empty prefix
14341 prefix(p);
14342 }
14343 }
14344
14345 void Assembler::prefix_rex2(Register dst, Address adr) {
14346 assert(!adr.index_needs_rex2(), "prefix(Register dst, Address adr) does not support handling of an X");
14347 int bits = 0;
14348 bits |= get_base_prefix_bits(adr.base());
14349 bits |= get_reg_prefix_bits(dst->encoding());
14350 prefix16(WREX2 | bits);
14351 }
14352
14353 void Assembler::prefix(Address adr, bool is_map1) {
14354 if (adr.base_needs_rex2() || adr.index_needs_rex2()) {
14355 prefix_rex2(adr, is_map1);
14356 return;
14357 }
14358 if (adr.base_needs_rex()) {
14359 if (adr.index_needs_rex()) {
14360 prefix(REX_XB);
14361 } else {
14362 prefix(REX_B);
14363 }
14364 } else {
14365 if (adr.index_needs_rex()) {
14366 prefix(REX_X);
14367 }
14368 }
14369 if (is_map1) emit_int8(0x0F);
14370 }
14371
14372 void Assembler::prefix_rex2(Address adr, bool is_map1) {
14373 int bits = is_map1 ? REX2BIT_M0 : 0;
14374 bits |= get_base_prefix_bits(adr.base());
14375 bits |= get_index_prefix_bits(adr.index());
14376 prefix16(WREX2 | bits);
14377 }
14378
14379 void Assembler::prefix(Address adr, Register reg, bool byteinst, bool is_map1) {
14380 if (reg->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) {
14381 prefix_rex2(adr, reg, byteinst, is_map1);
14382 return;
14383 }
14384 if (reg->encoding() < 8) {
14385 if (adr.base_needs_rex()) {
14386 if (adr.index_needs_rex()) {
14387 prefix(REX_XB);
14388 } else {
14389 prefix(REX_B);
14390 }
14391 } else {
14392 if (adr.index_needs_rex()) {
14393 prefix(REX_X);
14394 } else if (byteinst && reg->encoding() >= 4) {
14395 prefix(REX);
14396 }
14397 }
14398 } else {
14399 if (adr.base_needs_rex()) {
14400 if (adr.index_needs_rex()) {
14401 prefix(REX_RXB);
14402 } else {
14403 prefix(REX_RB);
14404 }
14405 } else {
14406 if (adr.index_needs_rex()) {
14407 prefix(REX_RX);
14408 } else {
14409 prefix(REX_R);
14410 }
14411 }
14412 }
14413 if (is_map1) emit_int8(0x0F);
14414 }
14415
14416 void Assembler::prefix_rex2(Address adr, Register reg, bool byteinst, bool is_map1) {
14417 int bits = is_map1 ? REX2BIT_M0 : 0;
14418 bits |= get_base_prefix_bits(adr.base());
14419 bits |= get_index_prefix_bits(adr.index());
14420 bits |= get_reg_prefix_bits(reg->encoding());
14421 prefix16(WREX2 | bits);
14422 }
14423
14424 void Assembler::prefix(Address adr, XMMRegister reg) {
14425 if (reg->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) {
14426 prefixq_rex2(adr, reg);
14427 return;
14428 }
14429 if (reg->encoding() < 8) {
14430 if (adr.base_needs_rex()) {
14431 if (adr.index_needs_rex()) {
14432 prefix(REX_XB);
14433 } else {
14434 prefix(REX_B);
14435 }
14436 } else {
14437 if (adr.index_needs_rex()) {
14438 prefix(REX_X);
14439 }
14440 }
14441 } else {
14442 if (adr.base_needs_rex()) {
14443 if (adr.index_needs_rex()) {
14444 prefix(REX_RXB);
14445 } else {
14446 prefix(REX_RB);
14447 }
14448 } else {
14449 if (adr.index_needs_rex()) {
14450 prefix(REX_RX);
14451 } else {
14452 prefix(REX_R);
14453 }
14454 }
14455 }
14456 }
14457
14458 void Assembler::prefix_rex2(Address adr, XMMRegister src) {
14459 int bits = 0;
14460 bits |= get_base_prefix_bits(adr.base());
14461 bits |= get_index_prefix_bits(adr.index());
14462 bits |= get_reg_prefix_bits(src->encoding());
14463 prefix16(WREX2 | bits);
14464 }
14465
14466 int Assembler::prefix_and_encode(int reg_enc, bool byteinst, bool is_map1) {
14467 if (reg_enc >= 16) {
14468 return prefix_and_encode_rex2(reg_enc, is_map1);
14469 }
14470 if (reg_enc >= 8) {
14471 prefix(REX_B);
14472 reg_enc -= 8;
14473 } else if (byteinst && reg_enc >= 4) {
14474 prefix(REX);
14475 }
14476 int opc_prefix = is_map1 ? 0x0F00 : 0;
14477 return opc_prefix | reg_enc;
14478 }
14479
14480 int Assembler::prefix_and_encode_rex2(int reg_enc, bool is_map1) {
14481 prefix16(WREX2 | (is_map1 ? REX2BIT_M0 : 0) | get_base_prefix_bits(reg_enc));
14482 return reg_enc & 0x7;
14483 }
14484
14485 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte, bool is_map1) {
14486 if (src_enc >= 16 || dst_enc >= 16) {
14487 return prefix_and_encode_rex2(dst_enc, src_enc, is_map1 ? REX2BIT_M0 : 0);
14488 }
14489 if (dst_enc < 8) {
14490 if (src_enc >= 8) {
14491 prefix(REX_B);
14492 src_enc -= 8;
14493 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
14494 prefix(REX);
14495 }
14496 } else {
14497 if (src_enc < 8) {
14498 prefix(REX_R);
14499 } else {
14500 prefix(REX_RB);
14501 src_enc -= 8;
14502 }
14503 dst_enc -= 8;
14504 }
14505 int opcode_prefix = is_map1 ? 0x0F00 : 0;
14506 return opcode_prefix | (dst_enc << 3 | src_enc);
14507 }
14508
14509 int Assembler::prefix_and_encode_rex2(int dst_enc, int src_enc, int init_bits) {
14510 int bits = init_bits;
14511 bits |= get_reg_prefix_bits(dst_enc);
14512 bits |= get_base_prefix_bits(src_enc);
14513 dst_enc &= 0x7;
14514 src_enc &= 0x7;
14515 prefix16(WREX2 | bits);
14516 return dst_enc << 3 | src_enc;
14517 }
14518
14519 bool Assembler::prefix_is_rex2(int prefix) {
14520 return (prefix & 0xFF00) == WREX2;
14521 }
14522
14523 int Assembler::get_prefixq_rex2(Address adr, bool is_map1) {
14524 assert(UseAPX, "APX features not enabled");
14525 int bits = REX2BIT_W;
14526 if (is_map1) bits |= REX2BIT_M0;
14527 bits |= get_base_prefix_bits(adr.base());
14528 bits |= get_index_prefix_bits(adr.index());
14529 return WREX2 | bits;
14530 }
14531
14532 int Assembler::get_prefixq(Address adr, bool is_map1) {
14533 if (adr.base_needs_rex2() || adr.index_needs_rex2()) {
14534 return get_prefixq_rex2(adr, is_map1);
14535 }
14536 int8_t prfx = get_prefixq(adr, rax);
14537 assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
14538 return is_map1 ? (((int16_t)prfx) << 8) | 0x0F : (int16_t)prfx;
14539 }
14540
14541 int Assembler::get_prefixq(Address adr, Register src, bool is_map1) {
14542 if (adr.base_needs_rex2() || adr.index_needs_rex2() || src->encoding() >= 16) {
14543 return get_prefixq_rex2(adr, src, is_map1);
14544 }
14545 int8_t prfx = (int8_t)(REX_W +
14546 ((int)adr.base_needs_rex()) +
14547 ((int)adr.index_needs_rex() << 1) +
14548 ((int)(src->encoding() >= 8) << 2));
14549 #ifdef ASSERT
14550 if (src->encoding() < 8) {
14551 if (adr.base_needs_rex()) {
14552 if (adr.index_needs_rex()) {
14553 assert(prfx == REX_WXB, "must be");
14554 } else {
14555 assert(prfx == REX_WB, "must be");
14556 }
14557 } else {
14558 if (adr.index_needs_rex()) {
14559 assert(prfx == REX_WX, "must be");
14560 } else {
14561 assert(prfx == REX_W, "must be");
14562 }
14563 }
14564 } else {
14565 if (adr.base_needs_rex()) {
14566 if (adr.index_needs_rex()) {
14567 assert(prfx == REX_WRXB, "must be");
14568 } else {
14569 assert(prfx == REX_WRB, "must be");
14570 }
14571 } else {
14572 if (adr.index_needs_rex()) {
14573 assert(prfx == REX_WRX, "must be");
14574 } else {
14575 assert(prfx == REX_WR, "must be");
14576 }
14577 }
14578 }
14579 #endif
14580 return is_map1 ? (((int16_t)prfx) << 8) | 0x0F : (int16_t)prfx;
14581 }
14582
14583 int Assembler::get_prefixq_rex2(Address adr, Register src, bool is_map1) {
14584 assert(UseAPX, "APX features not enabled");
14585 int bits = REX2BIT_W;
14586 if (is_map1) bits |= REX2BIT_M0;
14587 bits |= get_base_prefix_bits(adr.base());
14588 bits |= get_index_prefix_bits(adr.index());
14589 bits |= get_reg_prefix_bits(src->encoding());
14590 return WREX2 | bits;
14591 }
14592
14593 void Assembler::prefixq(Address adr) {
14594 if (adr.base_needs_rex2() || adr.index_needs_rex2()) {
14595 prefix16(get_prefixq_rex2(adr));
14596 } else {
14597 emit_int8(get_prefixq(adr));
14598 }
14599 }
14600
14601 void Assembler::prefixq(Address adr, Register src, bool is_map1) {
14602 if (adr.base_needs_rex2() || adr.index_needs_rex2() || src->encoding() >= 16) {
14603 prefix16(get_prefixq_rex2(adr, src, is_map1));
14604 } else {
14605 emit_int8(get_prefixq(adr, src));
14606 if (is_map1) emit_int8(0x0F);
14607 }
14608 }
14609
14610
14611 void Assembler::prefixq(Address adr, XMMRegister src) {
14612 if (src->encoding() >= 16 || adr.base_needs_rex2() || adr.index_needs_rex2()) {
14613 prefixq_rex2(adr, src);
14614 return;
14615 }
14616 if (src->encoding() < 8) {
14617 if (adr.base_needs_rex()) {
14618 if (adr.index_needs_rex()) {
14619 prefix(REX_WXB);
14620 } else {
14621 prefix(REX_WB);
14622 }
14623 } else {
14624 if (adr.index_needs_rex()) {
14625 prefix(REX_WX);
14626 } else {
14627 prefix(REX_W);
14628 }
14629 }
14630 } else {
14631 if (adr.base_needs_rex()) {
14632 if (adr.index_needs_rex()) {
14633 prefix(REX_WRXB);
14634 } else {
14635 prefix(REX_WRB);
14636 }
14637 } else {
14638 if (adr.index_needs_rex()) {
14639 prefix(REX_WRX);
14640 } else {
14641 prefix(REX_WR);
14642 }
14643 }
14644 }
14645 }
14646
14647 void Assembler::prefixq_rex2(Address adr, XMMRegister src) {
14648 int bits = REX2BIT_W;
14649 bits |= get_base_prefix_bits(adr.base());
14650 bits |= get_index_prefix_bits(adr.index());
14651 bits |= get_reg_prefix_bits(src->encoding());
14652 prefix16(WREX2 | bits);
14653 }
14654
14655 int Assembler::prefixq_and_encode(int reg_enc, bool is_map1) {
14656 if (reg_enc >= 16) {
14657 return prefixq_and_encode_rex2(reg_enc, is_map1);
14658 }
14659 if (reg_enc < 8) {
14660 prefix(REX_W);
14661 } else {
14662 prefix(REX_WB);
14663 reg_enc -= 8;
14664 }
14665 int opcode_prefix = is_map1 ? 0x0F00 : 0;
14666 return opcode_prefix | reg_enc;
14667 }
14668
14669
14670 int Assembler::prefixq_and_encode_rex2(int reg_enc, bool is_map1) {
14671 prefix16(WREX2 | REX2BIT_W | (is_map1 ? REX2BIT_M0: 0) | get_base_prefix_bits(reg_enc));
14672 return reg_enc & 0x7;
14673 }
14674
14675 int Assembler::prefixq_and_encode(int dst_enc, int src_enc, bool is_map1) {
14676 if (dst_enc >= 16 || src_enc >= 16) {
14677 return prefixq_and_encode_rex2(dst_enc, src_enc, is_map1);
14678 }
14679 if (dst_enc < 8) {
14680 if (src_enc < 8) {
14681 prefix(REX_W);
14682 } else {
14683 prefix(REX_WB);
14684 src_enc -= 8;
14685 }
14686 } else {
14687 if (src_enc < 8) {
14688 prefix(REX_WR);
14689 } else {
14690 prefix(REX_WRB);
14691 src_enc -= 8;
14692 }
14693 dst_enc -= 8;
14694 }
14695 int opcode_prefix = is_map1 ? 0x0F00 : 0;
14696 return opcode_prefix | (dst_enc << 3 | src_enc);
14697 }
14698
14699 int Assembler::prefixq_and_encode_rex2(int dst_enc, int src_enc, bool is_map1) {
14700 int init_bits = REX2BIT_W | (is_map1 ? REX2BIT_M0 : 0);
14701 return prefix_and_encode_rex2(dst_enc, src_enc, init_bits);
14702 }
14703
14704 void Assembler::emit_prefix_and_int8(int prefix, int b1) {
14705 if ((prefix & 0xFF00) == 0) {
14706 emit_int16(prefix, b1);
14707 } else {
14708 assert((prefix & 0xFF00) != WREX2 || UseAPX, "APX features not enabled");
14709 emit_int24((prefix & 0xFF00) >> 8, prefix & 0x00FF, b1);
14710 }
14711 }
14712
14713 void Assembler::adcq(Register dst, int32_t imm32) {
14714 (void) prefixq_and_encode(dst->encoding());
14715 emit_arith(0x81, 0xD0, dst, imm32);
14716 }
14717
14718 void Assembler::adcq(Register dst, Address src) {
14719 InstructionMark im(this);
14720 emit_prefix_and_int8(get_prefixq(src, dst), 0x13);
14721 emit_operand(dst, src, 0);
14722 }
14723
14724 void Assembler::adcq(Register dst, Register src) {
14725 (void) prefixq_and_encode(dst->encoding(), src->encoding());
14726 emit_arith(0x13, 0xC0, dst, src);
14727 }
14728
14729 void Assembler::addq(Address dst, int32_t imm32) {
14730 InstructionMark im(this);
14731 prefixq(dst);
14732 emit_arith_operand(0x81, rax, dst, imm32);
14733 }
14734
14735 void Assembler::eaddq(Register dst, Address src, int32_t imm32, bool no_flags) {
14736 InstructionMark im(this);
14737 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14738 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14739 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
14740 emit_arith_operand(0x81, rax, src, imm32);
14741 }
14742
14743 void Assembler::addq(Address dst, Register src) {
14744 InstructionMark im(this);
14745 emit_prefix_and_int8(get_prefixq(dst, src), 0x01);
14746 emit_operand(src, dst, 0);
14747 }
14748
14749 void Assembler::eaddq(Register dst, Address src1, Register src2, bool no_flags) {
14750 InstructionMark im(this);
14751 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
14752 }
14753
14754 void Assembler::addq(Register dst, int32_t imm32) {
14755 (void) prefixq_and_encode(dst->encoding());
14756 emit_arith(0x81, 0xC0, dst, imm32);
14757 }
14758
14759 void Assembler::eaddq(Register dst, Register src, int32_t imm32, bool no_flags) {
14760 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xC0, no_flags);
14761 }
14762
14763 void Assembler::addq(Register dst, Address src) {
14764 InstructionMark im(this);
14765 emit_prefix_and_int8(get_prefixq(src, dst), 0x03);
14766 emit_operand(dst, src, 0);
14767 }
14768
14769 void Assembler::eaddq(Register dst, Register src1, Address src2, bool no_flags) {
14770 InstructionMark im(this);
14771 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, no_flags);
14772 }
14773
14774 void Assembler::addq(Register dst, Register src) {
14775 (void) prefixq_and_encode(dst->encoding(), src->encoding());
14776 emit_arith(0x03, 0xC0, dst, src);
14777 }
14778
14779 void Assembler::eaddq(Register dst, Register src1, Register src2, bool no_flags) {
14780 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
14781 }
14782
14783 void Assembler::adcxq(Register dst, Register src) {
14784 //assert(VM_Version::supports_adx(), "adx instructions not supported");
14785 if (needs_rex2(dst, src)) {
14786 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14787 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true);
14788 emit_int16((unsigned char)0x66, (0xC0 | encode));
14789 } else {
14790 emit_int8(0x66);
14791 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
14792 emit_int32(0x0F,
14793 0x38,
14794 (unsigned char)0xF6,
14795 (0xC0 | encode));
14796 }
14797 }
14798
14799 void Assembler::eadcxq(Register dst, Register src1, Register src2) {
14800 if (is_demotable(false, dst->encoding(), src1->encoding())) {
14801 return adcxq(dst, src2);
14802 }
14803 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14804 int encode = emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */);
14805 emit_int16((unsigned char)0x66, (0xC0 | encode));
14806 }
14807
14808 void Assembler::adoxq(Register dst, Register src) {
14809 //assert(VM_Version::supports_adx(), "adx instructions not supported");
14810 if (needs_rex2(dst, src)) {
14811 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14812 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, true);
14813 emit_int16((unsigned char)0x66, (0xC0 | encode));
14814 } else {
14815 emit_int8((unsigned char)0xF3);
14816 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
14817 emit_int32(0x0F,
14818 0x38,
14819 (unsigned char)0xF6,
14820 (0xC0 | encode));
14821 }
14822 }
14823
14824 void Assembler::eadoxq(Register dst, Register src1, Register src2) {
14825 if (is_demotable(false, dst->encoding(), src1->encoding())) {
14826 return adoxq(dst, src2);
14827 }
14828 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14829 int encode = emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */);
14830 emit_int16((unsigned char)0x66, (0xC0 | encode));
14831 }
14832
14833 void Assembler::andq(Address dst, int32_t imm32) {
14834 InstructionMark im(this);
14835 prefixq(dst);
14836 emit_arith_operand(0x81, as_Register(4), dst, imm32);
14837 }
14838
14839 void Assembler::eandq(Register dst, Address src, int32_t imm32, bool no_flags) {
14840 InstructionMark im(this);
14841 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14842 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14843 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
14844 emit_arith_operand(0x81, as_Register(4), src, imm32);
14845 }
14846
14847 void Assembler::andq(Register dst, int32_t imm32) {
14848 (void) prefixq_and_encode(dst->encoding());
14849 emit_arith(0x81, 0xE0, dst, imm32);
14850 }
14851
14852 void Assembler::eandq(Register dst, Register src, int32_t imm32, bool no_flags) {
14853 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xE0, no_flags);
14854 }
14855
14856 void Assembler::andq(Register dst, Address src) {
14857 InstructionMark im(this);
14858 emit_prefix_and_int8(get_prefixq(src, dst), 0x23);
14859 emit_operand(dst, src, 0);
14860 }
14861
14862 void Assembler::eandq(Register dst, Register src1, Address src2, bool no_flags) {
14863 InstructionMark im(this);
14864 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, no_flags);
14865 }
14866
14867 void Assembler::andq(Register dst, Register src) {
14868 (void) prefixq_and_encode(dst->encoding(), src->encoding());
14869 emit_arith(0x23, 0xC0, dst, src);
14870 }
14871
14872 void Assembler::eandq(Register dst, Register src1, Register src2, bool no_flags) {
14873 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
14874 }
14875
14876 void Assembler::andq(Address dst, Register src) {
14877 InstructionMark im(this);
14878 emit_prefix_and_int8(get_prefixq(dst, src), 0x21);
14879 emit_operand(src, dst, 0);
14880 }
14881
14882 void Assembler::eandq(Register dst, Address src1, Register src2, bool no_flags) {
14883 InstructionMark im(this);
14884 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
14885 }
14886
14887 void Assembler::andnq(Register dst, Register src1, Register src2) {
14888 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14889 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14890 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
14891 emit_int16((unsigned char)0xF2, (0xC0 | encode));
14892 }
14893
14894 void Assembler::andnq(Register dst, Register src1, Address src2) {
14895 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14896 InstructionMark im(this);
14897 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14898 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14899 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
14900 emit_int8((unsigned char)0xF2);
14901 emit_operand(dst, src2, 0);
14902 }
14903
14904 void Assembler::bsfq(Register dst, Register src) {
14905 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
14906 emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
14907 }
14908
14909 void Assembler::bsrq(Register dst, Register src) {
14910 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
14911 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode);
14912 }
14913
14914 void Assembler::bswapq(Register reg) {
14915 int encode = prefixq_and_encode(reg->encoding(), true /* is_map1 */);
14916 emit_opcode_prefix_and_encoding((unsigned char)0xC8, encode);
14917 }
14918
14919 void Assembler::blsiq(Register dst, Register src) {
14920 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14921 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14922 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
14923 emit_int16((unsigned char)0xF3, (0xC0 | encode));
14924 }
14925
14926 void Assembler::blsiq(Register dst, Address src) {
14927 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14928 InstructionMark im(this);
14929 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14930 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14931 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
14932 emit_int8((unsigned char)0xF3);
14933 emit_operand(rbx, src, 0);
14934 }
14935
14936 void Assembler::blsmskq(Register dst, Register src) {
14937 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14938 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14939 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
14940 emit_int16((unsigned char)0xF3, (0xC0 | encode));
14941 }
14942
14943 void Assembler::blsmskq(Register dst, Address src) {
14944 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14945 InstructionMark im(this);
14946 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14947 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14948 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
14949 emit_int8((unsigned char)0xF3);
14950 emit_operand(rdx, src, 0);
14951 }
14952
14953 void Assembler::blsrq(Register dst, Register src) {
14954 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14955 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14956 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
14957 emit_int16((unsigned char)0xF3, (0xC0 | encode));
14958 }
14959
14960 void Assembler::blsrq(Register dst, Address src) {
14961 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
14962 InstructionMark im(this);
14963 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
14964 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
14965 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
14966 emit_int8((unsigned char)0xF3);
14967 emit_operand(rcx, src, 0);
14968 }
14969
14970 void Assembler::cdqq() {
14971 emit_int16(REX_W, (unsigned char)0x99);
14972 }
14973
14974 void Assembler::cdqe() {
14975 emit_int16(REX_W, (unsigned char)0x98);
14976 }
14977
14978 void Assembler::clflush(Address adr) {
14979 assert(VM_Version::supports_clflush(), "should do");
14980 prefix(adr, true /* is_map1 */);
14981 emit_int8((unsigned char)0xAE);
14982 emit_operand(rdi, adr, 0);
14983 }
14984
14985 void Assembler::clflushopt(Address adr) {
14986 assert(VM_Version::supports_clflushopt(), "should do!");
14987 // adr should be base reg only with no index or offset
14988 assert(adr.index() == noreg, "index should be noreg");
14989 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
14990 assert(adr.disp() == 0, "displacement should be 0");
14991 // instruction prefix is 0x66
14992 emit_int8(0x66);
14993 prefix(adr, true /* is_map1 */);
14994 // opcode family is 0x0F 0xAE
14995 emit_int8((unsigned char)0xAE);
14996 // extended opcode byte is 7 == rdi
14997 emit_operand(rdi, adr, 0);
14998 }
14999
15000 void Assembler::clwb(Address adr) {
15001 assert(VM_Version::supports_clwb(), "should do!");
15002 // adr should be base reg only with no index or offset
15003 assert(adr.index() == noreg, "index should be noreg");
15004 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
15005 assert(adr.disp() == 0, "displacement should be 0");
15006 // instruction prefix is 0x66
15007 emit_int8(0x66);
15008 prefix(adr, true /* is_map1 */);
15009 // opcode family is 0x0f 0xAE
15010 emit_int8((unsigned char)0xAE);
15011 // extended opcode byte is 6 == rsi
15012 emit_operand(rsi, adr, 0);
15013 }
15014
15015 void Assembler::cmovq(Condition cc, Register dst, Register src) {
15016 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15017 emit_opcode_prefix_and_encoding((0x40 | cc), 0xC0, encode);
15018 }
15019
15020 void Assembler::ecmovq(Condition cc, Register dst, Register src1, Register src2) {
15021 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x40 | cc, false /* no_flags */, true /* is_map1 */, true /* swap */);
15022 }
15023
15024 void Assembler::cmovq(Condition cc, Register dst, Address src) {
15025 InstructionMark im(this);
15026 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15027 emit_prefix_and_int8(prefix, (0x40 | cc));
15028 emit_operand(dst, src, 0);
15029 }
15030
15031 void Assembler::ecmovq(Condition cc, Register dst, Register src1, Address src2) {
15032 InstructionMark im(this);
15033 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, (0x40 | cc) , false /* no_flags */, true /* is_map1 */);
15034 }
15035
15036 void Assembler::cmpq(Address dst, int32_t imm32) {
15037 InstructionMark im(this);
15038 prefixq(dst);
15039 emit_arith_operand(0x81, as_Register(7), dst, imm32);
15040 }
15041
15042 void Assembler::cmpq(Register dst, int32_t imm32) {
15043 (void) prefixq_and_encode(dst->encoding());
15044 emit_arith(0x81, 0xF8, dst, imm32);
15045 }
15046
15047 void Assembler::cmpq(Address dst, Register src) {
15048 InstructionMark im(this);
15049 emit_prefix_and_int8(get_prefixq(dst, src), 0x39);
15050 emit_operand(src, dst, 0);
15051 }
15052
15053 void Assembler::cmpq(Register dst, Register src) {
15054 (void) prefixq_and_encode(dst->encoding(), src->encoding());
15055 emit_arith(0x3B, 0xC0, dst, src);
15056 }
15057
15058 void Assembler::cmpq(Register dst, Address src) {
15059 InstructionMark im(this);
15060 emit_prefix_and_int8(get_prefixq(src, dst), 0x3B);
15061 emit_operand(dst, src, 0);
15062 }
15063
15064 void Assembler::cmpxchgq(Register reg, Address adr) {
15065 InstructionMark im(this);
15066 int prefix = get_prefixq(adr, reg, true /* is_map1 */);
15067 emit_prefix_and_int8(prefix, (unsigned char)0xB1);
15068 emit_operand(reg, adr, 0);
15069 }
15070
15071 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
15072 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15073 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes, true);
15074 emit_int16(0x2A, (0xC0 | encode));
15075 }
15076
15077 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
15078 InstructionMark im(this);
15079 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15080 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
15081 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
15082 emit_int8(0x2A);
15083 emit_operand(dst, src, 0);
15084 }
15085
15086 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
15087 InstructionMark im(this);
15088 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15089 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
15090 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
15091 emit_int8(0x2A);
15092 emit_operand(dst, src, 0);
15093 }
15094
15095 void Assembler::cvttsd2siq(Register dst, Address src) {
15096 // F2 REX.W 0F 2C /r
15097 // CVTTSD2SI r64, xmm1/m64
15098 InstructionMark im(this);
15099 emit_int8((unsigned char)0xF2);
15100 prefixq(src, dst, true /* is_map1 */);
15101 emit_int8((unsigned char)0x2C);
15102 emit_operand(dst, src, 0);
15103 }
15104
15105 void Assembler::evcvttsd2sisl(Register dst, XMMRegister src) {
15106 assert(VM_Version::supports_avx10_2(), "");
15107 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15108 attributes.set_is_evex_instruction();
15109 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
15110 emit_int16(0x6D, (0xC0 | encode));
15111 }
15112
15113 void Assembler::evcvttsd2sisl(Register dst, Address src) {
15114 assert(VM_Version::supports_avx10_2(), "");
15115 InstructionMark im(this);
15116 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15117 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
15118 attributes.set_is_evex_instruction();
15119 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
15120 emit_int8((unsigned char)0x6D);
15121 emit_operand(dst, src, 0);
15122 }
15123
15124 void Assembler::evcvttsd2sisq(Register dst, XMMRegister src) {
15125 assert(VM_Version::supports_avx10_2(), "");
15126 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15127 attributes.set_is_evex_instruction();
15128 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
15129 emit_int16(0x6D, (0xC0 | encode));
15130 }
15131
15132 void Assembler::evcvttsd2sisq(Register dst, Address src) {
15133 assert(VM_Version::supports_avx10_2(), "");
15134 InstructionMark im(this);
15135 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15136 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
15137 attributes.set_is_evex_instruction();
15138 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
15139 emit_int8((unsigned char)0x6D);
15140 emit_operand(dst, src, 0);
15141 }
15142
15143 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
15144 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15145 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
15146 emit_int16(0x2C, (0xC0 | encode));
15147 }
15148
15149 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
15150 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15151 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
15152 emit_int16(0x2D, (0xC0 | encode));
15153 }
15154
15155 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
15156 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15157 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
15158 emit_int16(0x2C, (0xC0 | encode));
15159 }
15160
15161 void Assembler::decl(Register dst) {
15162 // Don't use it directly. Use MacroAssembler::decrementl() instead.
15163 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
15164 int encode = prefix_and_encode(dst->encoding());
15165 emit_int16((unsigned char)0xFF, (0xC8 | encode));
15166 }
15167
15168 void Assembler::edecl(Register dst, Register src, bool no_flags) {
15169 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15170 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15171 emit_int16((unsigned char)0xFF, (0xC8 | encode));
15172 }
15173
15174 void Assembler::decq(Register dst) {
15175 // Don't use it directly. Use MacroAssembler::decrementq() instead.
15176 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
15177 int encode = prefixq_and_encode(dst->encoding());
15178 emit_int16((unsigned char)0xFF, 0xC8 | encode);
15179 }
15180
15181 void Assembler::edecq(Register dst, Register src, bool no_flags) {
15182 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15183 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
15184 emit_int16((unsigned char)0xFF, (0xC8 | encode));
15185 }
15186
15187 void Assembler::decq(Address dst) {
15188 // Don't use it directly. Use MacroAssembler::decrementq() instead.
15189 InstructionMark im(this);
15190 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xFF);
15191 emit_operand(rcx, dst, 0);
15192 }
15193
15194 void Assembler::edecq(Register dst, Address src, bool no_flags) {
15195 InstructionMark im(this);
15196 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15197 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15198 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15199 emit_int8((unsigned char)0xFF);
15200 emit_operand(rcx, src, 0);
15201 }
15202
15203 // can't use REX2
15204 void Assembler::fxrstor(Address src) {
15205 InstructionMark im(this);
15206 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
15207 emit_operand(as_Register(1), src, 0);
15208 }
15209
15210 // can't use REX2
15211 void Assembler::xrstor(Address src) {
15212 InstructionMark im(this);
15213 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
15214 emit_operand(as_Register(5), src, 0);
15215 }
15216
15217 // can't use REX2
15218 void Assembler::fxsave(Address dst) {
15219 InstructionMark im(this);
15220 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
15221 emit_operand(as_Register(0), dst, 0);
15222 }
15223
15224 // cant use REX2
15225 void Assembler::xsave(Address dst) {
15226 InstructionMark im(this);
15227 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
15228 emit_operand(as_Register(4), dst, 0);
15229 }
15230
15231 void Assembler::idivq(Register src) {
15232 int encode = prefixq_and_encode(src->encoding());
15233 emit_int16((unsigned char)0xF7, (0xF8 | encode));
15234 }
15235
15236 void Assembler::eidivq(Register src, bool no_flags) {
15237 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15238 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15239 emit_int16((unsigned char)0xF7, (0xF8 | encode));
15240 }
15241
15242 void Assembler::divq(Register src) {
15243 int encode = prefixq_and_encode(src->encoding());
15244 emit_int16((unsigned char)0xF7, (0xF0 | encode));
15245 }
15246
15247 void Assembler::edivq(Register src, bool no_flags) {
15248 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15249 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15250 emit_int16((unsigned char)0xF7, (0xF0 | encode));
15251 }
15252
15253 void Assembler::imulq(Register dst, Register src) {
15254 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15255 emit_opcode_prefix_and_encoding((unsigned char)0xAF, 0xC0, encode);
15256 }
15257
15258 void Assembler::eimulq(Register dst, Register src, bool no_flags) {
15259 if (is_demotable(no_flags, dst->encoding(), src->encoding())) {
15260 return imulq(dst);
15261 }
15262 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15263 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15264 emit_int16((unsigned char)0xAF, (0xC0 | encode));
15265 }
15266
15267 void Assembler::eimulq(Register dst, Register src1, Register src2, bool no_flags) {
15268 emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */);
15269 }
15270
15271 void Assembler::imulq(Register src) {
15272 int encode = prefixq_and_encode(src->encoding());
15273 emit_int16((unsigned char)0xF7, (0xE8 | encode));
15274 }
15275
15276 void Assembler::eimulq(Register src, bool no_flags) {
15277 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15278 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15279 emit_int16((unsigned char)0xF7, (0xE8 | encode));
15280 }
15281
15282 void Assembler::imulq(Register dst, Address src, int32_t value) {
15283 InstructionMark im(this);
15284 prefixq(src, dst);
15285 if (is8bit(value)) {
15286 emit_int8((unsigned char)0x6B);
15287 emit_operand(dst, src, 1);
15288 emit_int8(value);
15289 } else {
15290 emit_int8((unsigned char)0x69);
15291 emit_operand(dst, src, 4);
15292 emit_int32(value);
15293 }
15294 }
15295
15296 void Assembler::eimulq(Register dst, Address src, int32_t value, bool no_flags) {
15297 InstructionMark im(this);
15298 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15299 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
15300 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15301 if (is8bit(value)) {
15302 emit_int8((unsigned char)0x6B);
15303 emit_operand(dst, src, 1);
15304 emit_int8(value);
15305 } else {
15306 emit_int8((unsigned char)0x69);
15307 emit_operand(dst, src, 4);
15308 emit_int32(value);
15309 }
15310 }
15311
15312 void Assembler::imulq(Register dst, Register src, int value) {
15313 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
15314 if (is8bit(value)) {
15315 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
15316 } else {
15317 emit_int16(0x69, (0xC0 | encode));
15318 emit_int32(value);
15319 }
15320 }
15321
15322 void Assembler::eimulq(Register dst, Register src, int value, bool no_flags) {
15323 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15324 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15325 if (is8bit(value)) {
15326 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
15327 } else {
15328 emit_int16(0x69, (0xC0 | encode));
15329 emit_int32(value);
15330 }
15331 }
15332
15333 void Assembler::imulq(Register dst, Address src) {
15334 InstructionMark im(this);
15335 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15336 emit_prefix_and_int8(prefix, (unsigned char)0xAF);
15337 emit_operand(dst, src, 0);
15338 }
15339
15340 void Assembler::eimulq(Register dst, Address src, bool no_flags) {
15341 InstructionMark im(this);
15342 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15343 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15344 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15345
15346 emit_int8((unsigned char)0xAF);
15347 emit_operand(dst, src, 0);
15348 }
15349
15350 void Assembler::eimulq(Register dst, Register src1, Address src2, bool no_flags) {
15351 InstructionMark im(this);
15352 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, (unsigned char)0xAF, no_flags, true /* is_map1 */);
15353 }
15354
15355 void Assembler::incl(Register dst) {
15356 // Don't use it directly. Use MacroAssembler::incrementl() instead.
15357 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
15358 int encode = prefix_and_encode(dst->encoding());
15359 emit_int16((unsigned char)0xFF, (0xC0 | encode));
15360 }
15361
15362 void Assembler::eincl(Register dst, Register src, bool no_flags) {
15363 // Don't use it directly. Use MacroAssembler::incrementl() instead.
15364 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
15365 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15366 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15367 emit_int16((unsigned char)0xFF, (0xC0 | encode));
15368 }
15369
15370 void Assembler::incq(Register dst) {
15371 // Don't use it directly. Use MacroAssembler::incrementq() instead.
15372 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
15373 int encode = prefixq_and_encode(dst->encoding());
15374 emit_int16((unsigned char)0xFF, (0xC0 | encode));
15375 }
15376
15377 void Assembler::eincq(Register dst, Register src, bool no_flags) {
15378 // Don't use it directly. Use MacroAssembler::incrementq() instead.
15379 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
15380 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15381 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
15382 emit_int16((unsigned char)0xFF, (0xC0 | encode));
15383 }
15384
15385 void Assembler::incq(Address dst) {
15386 // Don't use it directly. Use MacroAssembler::incrementq() instead.
15387 InstructionMark im(this);
15388 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xFF);
15389 emit_operand(rax, dst, 0);
15390 }
15391
15392 void Assembler::eincq(Register dst, Address src, bool no_flags) {
15393 // Don't use it directly. Use MacroAssembler::incrementq() instead.
15394 InstructionMark im(this);
15395 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15396 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15397 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15398 emit_int8((unsigned char) 0xFF);
15399 emit_operand(rax, src, 0);
15400 }
15401
15402 void Assembler::lea(Register dst, Address src) {
15403 leaq(dst, src);
15404 }
15405
15406 void Assembler::leaq(Register dst, Address src) {
15407 InstructionMark im(this);
15408 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x8D);
15409 emit_operand(dst, src, 0);
15410 }
15411
15412 void Assembler::mov64(Register dst, int64_t imm64) {
15413 InstructionMark im(this);
15414 int encode = prefixq_and_encode(dst->encoding());
15415 emit_int8(0xB8 | encode);
15416 emit_int64(imm64);
15417 }
15418
15419 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
15420 InstructionMark im(this);
15421 int encode = prefixq_and_encode(dst->encoding());
15422 emit_int8(0xB8 | encode);
15423 emit_data64(imm64, rtype, format);
15424 }
15425
15426 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
15427 InstructionMark im(this);
15428 int encode = prefixq_and_encode(dst->encoding());
15429 emit_int8(0xB8 | encode);
15430 emit_data64(imm64, rspec);
15431 }
15432
15433 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
15434 InstructionMark im(this);
15435 int encode = prefix_and_encode(dst->encoding());
15436 emit_int8(0xB8 | encode);
15437 emit_data((int)imm32, rspec, narrow_oop_operand);
15438 }
15439
15440 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
15441 InstructionMark im(this);
15442 prefix(dst);
15443 emit_int8((unsigned char)0xC7);
15444 emit_operand(rax, dst, 4);
15445 emit_data((int)imm32, rspec, narrow_oop_operand);
15446 }
15447
15448 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
15449 InstructionMark im(this);
15450 int encode = prefix_and_encode(src1->encoding());
15451 emit_int16((unsigned char)0x81, (0xF8 | encode));
15452 emit_data((int)imm32, rspec, narrow_oop_operand);
15453 }
15454
15455 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
15456 InstructionMark im(this);
15457 prefix(src1);
15458 emit_int8((unsigned char)0x81);
15459 emit_operand(rax, src1, 4);
15460 emit_data((int)imm32, rspec, narrow_oop_operand);
15461 }
15462
15463 void Assembler::lzcntq(Register dst, Register src) {
15464 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
15465 emit_int8((unsigned char)0xF3);
15466 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15467 emit_opcode_prefix_and_encoding((unsigned char)0xBD, 0xC0, encode);
15468 }
15469
15470 void Assembler::elzcntq(Register dst, Register src, bool no_flags) {
15471 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
15472 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15473 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15474 emit_int16((unsigned char)0xF5, (0xC0 | encode));
15475 }
15476
15477 void Assembler::lzcntq(Register dst, Address src) {
15478 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
15479 InstructionMark im(this);
15480 emit_int8((unsigned char)0xF3);
15481 prefixq(src, dst, true /* is_map1 */);
15482 emit_int8((unsigned char)0xBD);
15483 emit_operand(dst, src, 0);
15484 }
15485
15486 void Assembler::elzcntq(Register dst, Address src, bool no_flags) {
15487 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
15488 InstructionMark im(this);
15489 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15490 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15491 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15492 emit_int8((unsigned char)0xF5);
15493 emit_operand(dst, src, 0);
15494 }
15495
15496 void Assembler::movdq(XMMRegister dst, Register src) {
15497 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15498 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
15499 emit_int16(0x6E, (0xC0 | encode));
15500 }
15501
15502 void Assembler::movdq(Register dst, XMMRegister src) {
15503 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15504 // swap src/dst to get correct prefix
15505 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes, true);
15506 emit_int16(0x7E,
15507 (0xC0 | encode));
15508 }
15509
15510 void Assembler::movq(Register dst, Register src) {
15511 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
15512 emit_int16((unsigned char)0x8B,
15513 (0xC0 | encode));
15514 }
15515
15516 void Assembler::movq(Register dst, Address src) {
15517 InstructionMark im(this);
15518 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x8B);
15519 emit_operand(dst, src, 0);
15520 }
15521
15522 void Assembler::movq(Address dst, Register src) {
15523 InstructionMark im(this);
15524 emit_prefix_and_int8(get_prefixq(dst, src), (unsigned char)0x89);
15525 emit_operand(src, dst, 0);
15526 }
15527
15528 void Assembler::movq(Address dst, int32_t imm32) {
15529 InstructionMark im(this);
15530 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC7);
15531 emit_operand(as_Register(0), dst, 4);
15532 emit_int32(imm32);
15533 }
15534
15535 void Assembler::movq(Register dst, int32_t imm32) {
15536 int encode = prefixq_and_encode(dst->encoding());
15537 emit_int16((unsigned char)0xC7, (0xC0 | encode));
15538 emit_int32(imm32);
15539 }
15540
15541 void Assembler::movsbq(Register dst, Address src) {
15542 InstructionMark im(this);
15543 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15544 emit_prefix_and_int8(prefix, (unsigned char)0xBE);
15545 emit_operand(dst, src, 0);
15546 }
15547
15548 void Assembler::movsbq(Register dst, Register src) {
15549 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15550 emit_opcode_prefix_and_encoding((unsigned char)0xBE, 0xC0, encode);
15551 }
15552
15553 void Assembler::movslq(Address dst, int32_t imm32) {
15554 assert(is_simm32(imm32), "lost bits");
15555 InstructionMark im(this);
15556 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC7);
15557 emit_operand(rax, dst, 4);
15558 emit_int32(imm32);
15559 }
15560
15561 void Assembler::movslq(Register dst, Address src) {
15562 InstructionMark im(this);
15563 emit_prefix_and_int8(get_prefixq(src, dst), 0x63);
15564 emit_operand(dst, src, 0);
15565 }
15566
15567 void Assembler::movslq(Register dst, Register src) {
15568 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
15569 emit_int16(0x63, (0xC0 | encode));
15570 }
15571
15572 void Assembler::movswq(Register dst, Address src) {
15573 InstructionMark im(this);
15574 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15575 emit_prefix_and_int8(prefix, (unsigned char)0xBF);
15576 emit_operand(dst, src, 0);
15577 }
15578
15579 void Assembler::movswq(Register dst, Register src) {
15580 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15581 emit_opcode_prefix_and_encoding((unsigned char)0xBF, 0xC0, encode);
15582 }
15583
15584 void Assembler::movzbq(Register dst, Address src) {
15585 InstructionMark im(this);
15586 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15587 emit_prefix_and_int8(prefix, (unsigned char)0xB6);
15588 emit_operand(dst, src, 0);
15589 }
15590
15591 void Assembler::movzbq(Register dst, Register src) {
15592 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15593 emit_opcode_prefix_and_encoding((unsigned char)0xB6, 0xC0, encode);
15594 }
15595
15596 void Assembler::movzwq(Register dst, Address src) {
15597 InstructionMark im(this);
15598 int prefix = get_prefixq(src, dst, true /* is_map1 */);
15599 emit_prefix_and_int8(prefix, (unsigned char)0xB7);
15600 emit_operand(dst, src, 0);
15601 }
15602
15603 void Assembler::movzwq(Register dst, Register src) {
15604 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15605 emit_opcode_prefix_and_encoding((unsigned char)0xB7, 0xC0, encode);
15606 }
15607
15608 void Assembler::mulq(Address src) {
15609 InstructionMark im(this);
15610 emit_prefix_and_int8(get_prefixq(src), (unsigned char)0xF7);
15611 emit_operand(rsp, src, 0);
15612 }
15613
15614 void Assembler::emulq(Address src, bool no_flags) {
15615 InstructionMark im(this);
15616 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15617 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15618 eevex_prefix_nf(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15619 emit_int8(0xF7);
15620 emit_operand(rsp, src, 0);
15621 }
15622
15623 void Assembler::mulq(Register src) {
15624 int encode = prefixq_and_encode(src->encoding());
15625 emit_int16((unsigned char)0xF7, (0xE0 | encode));
15626 }
15627
15628 void Assembler::emulq(Register src, bool no_flags) {
15629 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15630 int encode = eevex_prefix_and_encode_nf(0, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15631 emit_int16((unsigned char)0xF7, (0xE0 | encode));
15632 }
15633
15634 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
15635 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
15636 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15637 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
15638 emit_int16((unsigned char)0xF6, (0xC0 | encode));
15639 }
15640
15641 void Assembler::negq(Register dst) {
15642 int encode = prefixq_and_encode(dst->encoding());
15643 emit_int16((unsigned char)0xF7, (0xD8 | encode));
15644 }
15645
15646 void Assembler::enegq(Register dst, Register src, bool no_flags) {
15647 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15648 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
15649 emit_int16((unsigned char)0xF7, (0xD8 | encode));
15650 }
15651
15652 void Assembler::negq(Address dst) {
15653 InstructionMark im(this);
15654 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xF7);
15655 emit_operand(as_Register(3), dst, 0);
15656 }
15657
15658 void Assembler::enegq(Register dst, Address src, bool no_flags) {
15659 InstructionMark im(this);
15660 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15661 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15662 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15663 emit_int8((unsigned char)0xF7);
15664 emit_operand(as_Register(3), src, 0);
15665 }
15666
15667 void Assembler::notq(Register dst) {
15668 int encode = prefixq_and_encode(dst->encoding());
15669 emit_int16((unsigned char)0xF7, (0xD0 | encode));
15670 }
15671
15672 void Assembler::enotq(Register dst, Register src) {
15673 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15674 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */);
15675 emit_int16((unsigned char)0xF7, (0xD0 | encode));
15676 }
15677
15678 void Assembler::btq(Register dst, Register src) {
15679 int encode = prefixq_and_encode(src->encoding(), dst->encoding(), true /* is_map1 */);
15680 emit_opcode_prefix_and_encoding((unsigned char)0xA3, 0xC0, encode);
15681 }
15682
15683 void Assembler::btq(Register src, int imm8) {
15684 assert(isByte(imm8), "not a byte");
15685 int encode = prefixq_and_encode(src->encoding(), true /* is_map1 */);
15686 emit_opcode_prefix_and_encoding((unsigned char)0xBA, 0xE0, encode);
15687 emit_int8(imm8);
15688 }
15689
15690 void Assembler::btsq(Address dst, int imm8) {
15691 assert(isByte(imm8), "not a byte");
15692 InstructionMark im(this);
15693 int prefix = get_prefixq(dst, true /* is_map1 */);
15694 emit_prefix_and_int8(prefix, (unsigned char)0xBA);
15695 emit_operand(rbp /* 5 */, dst, 1);
15696 emit_int8(imm8);
15697 }
15698
15699 void Assembler::btrq(Address dst, int imm8) {
15700 assert(isByte(imm8), "not a byte");
15701 InstructionMark im(this);
15702 int prefix = get_prefixq(dst, true /* is_map1 */);
15703 emit_prefix_and_int8(prefix, (unsigned char)0xBA);
15704 emit_operand(rsi /* 6 */, dst, 1);
15705 emit_int8(imm8);
15706 }
15707
15708 void Assembler::orq(Address dst, int32_t imm32) {
15709 InstructionMark im(this);
15710 prefixq(dst);
15711 emit_arith_operand(0x81, as_Register(1), dst, imm32);
15712 }
15713
15714 void Assembler::eorq(Register dst, Address src, int32_t imm32, bool no_flags) {
15715 InstructionMark im(this);
15716 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15717 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15718 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15719 emit_arith_operand(0x81, as_Register(1), src, imm32);
15720 }
15721
15722 void Assembler::orq(Address dst, Register src) {
15723 InstructionMark im(this);
15724 emit_prefix_and_int8(get_prefixq(dst, src), (unsigned char)0x09);
15725 emit_operand(src, dst, 0);
15726 }
15727
15728 void Assembler::eorq(Register dst, Address src1, Register src2, bool no_flags) {
15729 InstructionMark im(this);
15730 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
15731 }
15732
15733 void Assembler::orq(Register dst, int32_t imm32) {
15734 (void) prefixq_and_encode(dst->encoding());
15735 emit_arith(0x81, 0xC8, dst, imm32);
15736 }
15737
15738 void Assembler::eorq(Register dst, Register src, int32_t imm32, bool no_flags) {
15739 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xC8, no_flags);
15740 }
15741
15742 void Assembler::orq_imm32(Register dst, int32_t imm32) {
15743 (void) prefixq_and_encode(dst->encoding());
15744 emit_arith_imm32(0x81, 0xC8, dst, imm32);
15745 }
15746
15747 void Assembler::eorq_imm32(Register dst, Register src, int32_t imm32, bool no_flags) {
15748 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15749 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
15750 emit_arith_imm32(0x81, 0xC8, src, imm32);
15751 }
15752
15753 void Assembler::orq(Register dst, Address src) {
15754 InstructionMark im(this);
15755 emit_prefix_and_int8(get_prefixq(src, dst), 0x0B);
15756 emit_operand(dst, src, 0);
15757 }
15758
15759 void Assembler::eorq(Register dst, Register src1, Address src2, bool no_flags) {
15760 InstructionMark im(this);
15761 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, no_flags);
15762 }
15763
15764 void Assembler::orq(Register dst, Register src) {
15765 (void) prefixq_and_encode(dst->encoding(), src->encoding());
15766 emit_arith(0x0B, 0xC0, dst, src);
15767 }
15768
15769 void Assembler::eorq(Register dst, Register src1, Register src2, bool no_flags) {
15770 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
15771 }
15772 void Assembler::popcntq(Register dst, Address src) {
15773 assert(VM_Version::supports_popcnt(), "must support");
15774 InstructionMark im(this);
15775 emit_int8((unsigned char)0xF3);
15776 emit_prefix_and_int8(get_prefixq(src, dst, true /* is_map1 */), (unsigned char) 0xB8);
15777 emit_operand(dst, src, 0);
15778 }
15779
15780 void Assembler::epopcntq(Register dst, Address src, bool no_flags) {
15781 assert(VM_Version::supports_popcnt(), "must support");
15782 InstructionMark im(this);
15783 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15784 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
15785 eevex_prefix_nf(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15786 emit_int8((unsigned char) 0x88);
15787 emit_operand(dst, src, 0);
15788 }
15789
15790 void Assembler::popcntq(Register dst, Register src) {
15791 assert(VM_Version::supports_popcnt(), "must support");
15792 emit_int8((unsigned char)0xF3);
15793 int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
15794 emit_opcode_prefix_and_encoding((unsigned char)0xB8, 0xC0, encode);
15795 }
15796
15797 void Assembler::epopcntq(Register dst, Register src, bool no_flags) {
15798 assert(VM_Version::supports_popcnt(), "must support");
15799 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
15800 int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
15801 emit_int16((unsigned char)0x88, (0xC0 | encode));
15802 }
15803
15804 void Assembler::popq(Address dst) {
15805 InstructionMark im(this);
15806 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0x8F);
15807 emit_operand(rax, dst, 0);
15808 }
15809
15810 void Assembler::popq(Register dst) {
15811 int encode = prefix_and_encode(dst->encoding());
15812 emit_int8((unsigned char)0x58 | encode);
15813 }
15814
15815 // Precomputable: popa, pusha, vzeroupper
15816
15817 // The result of these routines are invariant from one invocation to another
15818 // invocation for the duration of a run. Caching the result on bootstrap
15819 // and copying it out on subsequent invocations can thus be beneficial
15820 static bool precomputed = false;
15821
15822 static u_char* popa_code = nullptr;
15823 static int popa_len = 0;
15824
15825 static u_char* pusha_code = nullptr;
15826 static int pusha_len = 0;
15827
15828 static u_char* vzup_code = nullptr;
15829 static int vzup_len = 0;
15830
15831 void Assembler::precompute_instructions() {
15832 assert(!Universe::is_fully_initialized(), "must still be single threaded");
15833 guarantee(!precomputed, "only once");
15834 precomputed = true;
15835 ResourceMark rm;
15836
15837 // Make a temporary buffer big enough for the routines we're capturing
15838 int size = UseAPX ? 512 : 256;
15839 char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
15840 CodeBuffer buffer((address)tmp_code, size);
15841 MacroAssembler masm(&buffer);
15842
15843 address begin_popa = masm.code_section()->end();
15844 masm.popa_uncached();
15845 address end_popa = masm.code_section()->end();
15846 masm.pusha_uncached();
15847 address end_pusha = masm.code_section()->end();
15848 masm.vzeroupper_uncached();
15849 address end_vzup = masm.code_section()->end();
15850
15851 // Save the instructions to permanent buffers.
15852 popa_len = (int)(end_popa - begin_popa);
15853 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
15854 memcpy(popa_code, begin_popa, popa_len);
15855
15856 pusha_len = (int)(end_pusha - end_popa);
15857 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
15858 memcpy(pusha_code, end_popa, pusha_len);
15859
15860 vzup_len = (int)(end_vzup - end_pusha);
15861 if (vzup_len > 0) {
15862 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
15863 memcpy(vzup_code, end_pusha, vzup_len);
15864 } else {
15865 vzup_code = pusha_code; // dummy
15866 }
15867
15868 assert(masm.code()->total_oop_size() == 0 &&
15869 masm.code()->total_metadata_size() == 0 &&
15870 masm.code()->total_relocation_size() == 0,
15871 "pre-computed code can't reference oops, metadata or contain relocations");
15872 }
15873
15874 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
15875 assert(src != nullptr, "code to copy must have been pre-computed");
15876 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
15877 address end = code_section->end();
15878 memcpy(end, src, src_len);
15879 code_section->set_end(end + src_len);
15880 }
15881
15882
15883 // Does not actually store the value of rsp on the stack.
15884 // The slot for rsp just contains an arbitrary value.
15885 void Assembler::pusha() { // 64bit
15886 emit_copy(code_section(), pusha_code, pusha_len);
15887 }
15888
15889 // Does not actually store the value of rsp on the stack.
15890 // The slot for rsp just contains an arbitrary value.
15891 void Assembler::pusha_uncached() { // 64bit
15892 if (UseAPX) {
15893 // Data being pushed by PUSH2 must be 16B-aligned on the stack, for this push rax upfront
15894 // and use it as a temporary register for stack alignment.
15895 pushp(rax);
15896 // Move original stack pointer to RAX and align stack pointer to 16B boundary.
15897 movq(rax, rsp);
15898 andq(rsp, -(StackAlignmentInBytes));
15899 // Push pair of original stack pointer along with remaining registers
15900 // at 16B aligned boundary.
15901 push2p(rax, r31);
15902 // Restore the original contents of RAX register.
15903 movq(rax, Address(rax));
15904 push2p(r30, r29);
15905 push2p(r28, r27);
15906 push2p(r26, r25);
15907 push2p(r24, r23);
15908 push2p(r22, r21);
15909 push2p(r20, r19);
15910 push2p(r18, r17);
15911 push2p(r16, r15);
15912 push2p(r14, r13);
15913 push2p(r12, r11);
15914 push2p(r10, r9);
15915 push2p(r8, rdi);
15916 push2p(rsi, rbp);
15917 push2p(rbx, rdx);
15918 // To maintain 16 byte alignment after rcx is pushed.
15919 subq(rsp, 8);
15920 pushp(rcx);
15921 } else {
15922 subq(rsp, 16 * wordSize);
15923 movq(Address(rsp, 15 * wordSize), rax);
15924 movq(Address(rsp, 14 * wordSize), rcx);
15925 movq(Address(rsp, 13 * wordSize), rdx);
15926 movq(Address(rsp, 12 * wordSize), rbx);
15927 // Skip rsp as the value is normally not used. There are a few places where
15928 // the original value of rsp needs to be known but that can be computed
15929 // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
15930 // FIXME: For APX any such direct access should also consider EGPR size
15931 // during address compution.
15932 movq(Address(rsp, 10 * wordSize), rbp);
15933 movq(Address(rsp, 9 * wordSize), rsi);
15934 movq(Address(rsp, 8 * wordSize), rdi);
15935 movq(Address(rsp, 7 * wordSize), r8);
15936 movq(Address(rsp, 6 * wordSize), r9);
15937 movq(Address(rsp, 5 * wordSize), r10);
15938 movq(Address(rsp, 4 * wordSize), r11);
15939 movq(Address(rsp, 3 * wordSize), r12);
15940 movq(Address(rsp, 2 * wordSize), r13);
15941 movq(Address(rsp, wordSize), r14);
15942 movq(Address(rsp, 0), r15);
15943 }
15944 }
15945
15946 void Assembler::popa() { // 64bit
15947 emit_copy(code_section(), popa_code, popa_len);
15948 }
15949
15950 void Assembler::popa_uncached() { // 64bit
15951 if (UseAPX) {
15952 popp(rcx);
15953 addq(rsp, 8);
15954 // Data being popped by POP2 must be 16B-aligned on the stack.
15955 pop2p(rdx, rbx);
15956 pop2p(rbp, rsi);
15957 pop2p(rdi, r8);
15958 pop2p(r9, r10);
15959 pop2p(r11, r12);
15960 pop2p(r13, r14);
15961 pop2p(r15, r16);
15962 pop2p(r17, r18);
15963 pop2p(r19, r20);
15964 pop2p(r21, r22);
15965 pop2p(r23, r24);
15966 pop2p(r25, r26);
15967 pop2p(r27, r28);
15968 pop2p(r29, r30);
15969 // Popped value in RAX holds original unaligned stack pointer.
15970 pop2p(r31, rax);
15971 // Reinstantiate original stack pointer.
15972 movq(rsp, rax);
15973 popp(rax);
15974 } else {
15975 movq(r15, Address(rsp, 0));
15976 movq(r14, Address(rsp, wordSize));
15977 movq(r13, Address(rsp, 2 * wordSize));
15978 movq(r12, Address(rsp, 3 * wordSize));
15979 movq(r11, Address(rsp, 4 * wordSize));
15980 movq(r10, Address(rsp, 5 * wordSize));
15981 movq(r9, Address(rsp, 6 * wordSize));
15982 movq(r8, Address(rsp, 7 * wordSize));
15983 movq(rdi, Address(rsp, 8 * wordSize));
15984 movq(rsi, Address(rsp, 9 * wordSize));
15985 movq(rbp, Address(rsp, 10 * wordSize));
15986 // Skip rsp as it is restored automatically to the value
15987 // before the corresponding pusha when popa is done.
15988 movq(rbx, Address(rsp, 12 * wordSize));
15989 movq(rdx, Address(rsp, 13 * wordSize));
15990 movq(rcx, Address(rsp, 14 * wordSize));
15991 movq(rax, Address(rsp, 15 * wordSize));
15992
15993 addq(rsp, 16 * wordSize);
15994 }
15995 }
15996
15997 void Assembler::vzeroupper() {
15998 emit_copy(code_section(), vzup_code, vzup_len);
15999 }
16000
16001 void Assembler::vzeroall() {
16002 assert(VM_Version::supports_avx(), "requires AVX");
16003 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
16004 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
16005 emit_int8(0x77);
16006 }
16007
16008 void Assembler::pushq(Address src) {
16009 InstructionMark im(this);
16010 emit_prefix_and_int8(get_prefixq(src), (unsigned char)0xFF);
16011 emit_operand(rsi, src, 0);
16012 }
16013
16014 void Assembler::rclq(Register dst, int imm8) {
16015 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16016 int encode = prefixq_and_encode(dst->encoding());
16017 if (imm8 == 1) {
16018 emit_int16((unsigned char)0xD1, (0xD0 | encode));
16019 } else {
16020 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
16021 }
16022 }
16023
16024 void Assembler::erclq(Register dst, Register src, int imm8) {
16025 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16026 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16027 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */);
16028 if (imm8 == 1) {
16029 emit_int16((unsigned char)0xD1, (0xD0 | encode));
16030 } else {
16031 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
16032 }
16033 }
16034
16035 void Assembler::rcrq(Register dst, int imm8) {
16036 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16037 int encode = prefixq_and_encode(dst->encoding());
16038 if (imm8 == 1) {
16039 emit_int16((unsigned char)0xD1, (0xD8 | encode));
16040 } else {
16041 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
16042 }
16043 }
16044
16045 void Assembler::ercrq(Register dst, Register src, int imm8) {
16046 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16047 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16048 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, false /* no_flags */, true /* use_prefixq */);
16049 if (imm8 == 1) {
16050 emit_int16((unsigned char)0xD1, (0xD8 | encode));
16051 } else {
16052 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
16053 }
16054 }
16055
16056 void Assembler::rorxl(Register dst, Register src, int imm8) {
16057 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
16058 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16059 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes, true);
16060 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
16061 }
16062
16063 void Assembler::rorxl(Register dst, Address src, int imm8) {
16064 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
16065 InstructionMark im(this);
16066 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16067 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
16068 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
16069 emit_int8((unsigned char)0xF0);
16070 emit_operand(dst, src, 1);
16071 emit_int8(imm8);
16072 }
16073
16074 void Assembler::rorxq(Register dst, Register src, int imm8) {
16075 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
16076 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16077 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes, true);
16078 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
16079 }
16080
16081 void Assembler::rorxq(Register dst, Address src, int imm8) {
16082 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
16083 InstructionMark im(this);
16084 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16085 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16086 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
16087 emit_int8((unsigned char)0xF0);
16088 emit_operand(dst, src, 1);
16089 emit_int8(imm8);
16090 }
16091
16092 void Assembler::salq(Address dst, int imm8) {
16093 InstructionMark im(this);
16094 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16095 if (imm8 == 1) {
16096 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1);
16097 emit_operand(as_Register(4), dst, 0);
16098 }
16099 else {
16100 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1);
16101 emit_operand(as_Register(4), dst, 1);
16102 emit_int8(imm8);
16103 }
16104 }
16105
16106 void Assembler::esalq(Register dst, Address src, int imm8, bool no_flags) {
16107 InstructionMark im(this);
16108 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16109 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16110 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16111 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16112 if (imm8 == 1) {
16113 emit_int8((unsigned char)0xD1);
16114 emit_operand(as_Register(4), src, 0);
16115 }
16116 else {
16117 emit_int8((unsigned char)0xC1);
16118 emit_operand(as_Register(4), src, 1);
16119 emit_int8(imm8);
16120 }
16121 }
16122
16123 void Assembler::salq(Address dst) {
16124 InstructionMark im(this);
16125 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3);
16126 emit_operand(as_Register(4), dst, 0);
16127 }
16128
16129 void Assembler::esalq(Register dst, Address src, bool no_flags) {
16130 InstructionMark im(this);
16131 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16132 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16133 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16134 emit_int8((unsigned char)0xD3);
16135 emit_operand(as_Register(4), src, 0);
16136 }
16137
16138 void Assembler::salq(Register dst, int imm8) {
16139 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16140 int encode = prefixq_and_encode(dst->encoding());
16141 if (imm8 == 1) {
16142 emit_int16((unsigned char)0xD1, (0xE0 | encode));
16143 } else {
16144 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
16145 }
16146 }
16147
16148 void Assembler::esalq(Register dst, Register src, int imm8, bool no_flags) {
16149 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16150 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16151 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16152 if (imm8 == 1) {
16153 emit_int16((unsigned char)0xD1, (0xE0 | encode));
16154 } else {
16155 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
16156 }
16157 }
16158
16159 void Assembler::salq(Register dst) {
16160 int encode = prefixq_and_encode(dst->encoding());
16161 emit_int16((unsigned char)0xD3, (0xE0 | encode));
16162 }
16163
16164 void Assembler::esalq(Register dst, Register src, bool no_flags) {
16165 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16166 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16167 emit_int16((unsigned char)0xD3, (0xE0 | encode));
16168 }
16169
16170 void Assembler::sarq(Address dst, int imm8) {
16171 InstructionMark im(this);
16172 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16173 if (imm8 == 1) {
16174 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1);
16175 emit_operand(as_Register(7), dst, 0);
16176 }
16177 else {
16178 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1);
16179 emit_operand(as_Register(7), dst, 1);
16180 emit_int8(imm8);
16181 }
16182 }
16183
16184 void Assembler::esarq(Register dst, Address src, int imm8, bool no_flags) {
16185 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16186 InstructionMark im(this);
16187 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16188 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16189 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16190 if (imm8 == 1) {
16191 emit_int8((unsigned char)0xD1);
16192 emit_operand(as_Register(7), src, 0);
16193 }
16194 else {
16195 emit_int8((unsigned char)0xC1);
16196 emit_operand(as_Register(7), src, 1);
16197 emit_int8(imm8);
16198 }
16199 }
16200
16201 void Assembler::sarq(Address dst) {
16202 InstructionMark im(this);
16203 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3);
16204 emit_operand(as_Register(7), dst, 0);
16205 }
16206
16207 void Assembler::esarq(Register dst, Address src, bool no_flags) {
16208 InstructionMark im(this);
16209 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16210 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16211 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16212 emit_int8((unsigned char)0xD3);
16213 emit_operand(as_Register(7), src, 0);
16214 }
16215
16216 void Assembler::sarq(Register dst, int imm8) {
16217 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16218 int encode = prefixq_and_encode(dst->encoding());
16219 if (imm8 == 1) {
16220 emit_int16((unsigned char)0xD1, (0xF8 | encode));
16221 } else {
16222 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
16223 }
16224 }
16225
16226 void Assembler::esarq(Register dst, Register src, int imm8, bool no_flags) {
16227 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16228 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16229 if (imm8 == 1) {
16230 emit_int16((unsigned char)0xD1, (0xF8 | encode));
16231 } else {
16232 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
16233 }
16234 }
16235
16236 void Assembler::sarq(Register dst) {
16237 int encode = prefixq_and_encode(dst->encoding());
16238 emit_int16((unsigned char)0xD3, (0xF8 | encode));
16239 }
16240
16241 void Assembler::esarq(Register dst, Register src, bool no_flags) {
16242 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16243 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16244 emit_int16((unsigned char)0xD3, (0xF8 | encode));
16245 }
16246
16247 void Assembler::sbbq(Address dst, int32_t imm32) {
16248 InstructionMark im(this);
16249 prefixq(dst);
16250 emit_arith_operand(0x81, rbx, dst, imm32);
16251 }
16252
16253 void Assembler::sbbq(Register dst, int32_t imm32) {
16254 (void) prefixq_and_encode(dst->encoding());
16255 emit_arith(0x81, 0xD8, dst, imm32);
16256 }
16257
16258 void Assembler::sbbq(Register dst, Address src) {
16259 InstructionMark im(this);
16260 emit_prefix_and_int8(get_prefixq(src, dst), 0x1B);
16261 emit_operand(dst, src, 0);
16262 }
16263
16264 void Assembler::sbbq(Register dst, Register src) {
16265 (void) prefixq_and_encode(dst->encoding(), src->encoding());
16266 emit_arith(0x1B, 0xC0, dst, src);
16267 }
16268
16269 void Assembler::shlq(Register dst, int imm8) {
16270 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16271 int encode = prefixq_and_encode(dst->encoding());
16272 if (imm8 == 1) {
16273 emit_int16((unsigned char)0xD1, (0xE0 | encode));
16274 } else {
16275 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
16276 }
16277 }
16278
16279 void Assembler::eshlq(Register dst, Register src, int imm8, bool no_flags) {
16280 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16281 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16282 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16283 if (imm8 == 1 ) {
16284 emit_int16((unsigned char)0xD1, (0xE0 | encode));
16285 } else {
16286 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
16287 }
16288 }
16289
16290 void Assembler::shlq(Register dst) {
16291 int encode = prefixq_and_encode(dst->encoding());
16292 emit_int16((unsigned char)0xD3, (0xE0 | encode));
16293 }
16294
16295 void Assembler::eshlq(Register dst, Register src, bool no_flags) {
16296 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16297 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16298 emit_int16((unsigned char)0xD3, (0xE0 | encode));
16299 }
16300
16301 void Assembler::shrq(Register dst, int imm8) {
16302 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16303 int encode = prefixq_and_encode(dst->encoding());
16304 if (imm8 == 1) {
16305 emit_int16((unsigned char)0xD1, (0xE8 | encode));
16306 }
16307 else {
16308 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
16309 }
16310 }
16311
16312 void Assembler::eshrq(Register dst, Register src, int imm8, bool no_flags) {
16313 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16314 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16315 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16316 if (imm8 == 1) {
16317 emit_int16((unsigned char)0xD1, (0xE8 | encode));
16318 }
16319 else {
16320 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
16321 }
16322 }
16323
16324 void Assembler::shrq(Register dst) {
16325 int encode = prefixq_and_encode(dst->encoding());
16326 emit_int16((unsigned char)0xD3, 0xE8 | encode);
16327 }
16328
16329 void Assembler::eshrq(Register dst, Register src, bool no_flags) {
16330 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16331 int encode = emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16332 emit_int16((unsigned char)0xD3, (0xE8 | encode));
16333 }
16334
16335 void Assembler::shrq(Address dst) {
16336 InstructionMark im(this);
16337 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD3);
16338 emit_operand(as_Register(5), dst, 0);
16339 }
16340
16341 void Assembler::eshrq(Register dst, Address src, bool no_flags) {
16342 InstructionMark im(this);
16343 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16344 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16345 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16346 emit_int8((unsigned char)0xD3);
16347 emit_operand(as_Register(5), src, 0);
16348 }
16349
16350 void Assembler::shrq(Address dst, int imm8) {
16351 InstructionMark im(this);
16352 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16353 if (imm8 == 1) {
16354 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xD1);
16355 emit_operand(as_Register(5), dst, 0);
16356 }
16357 else {
16358 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xC1);
16359 emit_operand(as_Register(5), dst, 1);
16360 emit_int8(imm8);
16361 }
16362 }
16363
16364 void Assembler::eshrq(Register dst, Address src, int imm8, bool no_flags) {
16365 InstructionMark im(this);
16366 assert(isShiftCount(imm8 >> 1), "illegal shift count");
16367 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16368 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16369 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16370 if (imm8 == 1) {
16371 emit_int8((unsigned char)0xD1);
16372 emit_operand(as_Register(5), src, 0);
16373 }
16374 else {
16375 emit_int8((unsigned char)0xC1);
16376 emit_operand(as_Register(5), src, 1);
16377 emit_int8(imm8);
16378 }
16379 }
16380
16381 void Assembler::subq(Address dst, int32_t imm32) {
16382 InstructionMark im(this);
16383 prefixq(dst);
16384 emit_arith_operand(0x81, rbp, dst, imm32);
16385 }
16386
16387 void Assembler::esubq(Register dst, Address src, int32_t imm32, bool no_flags) {
16388 InstructionMark im(this);
16389 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16390 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16391 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16392 emit_arith_operand(0x81, rbp, src, imm32);
16393 }
16394
16395 void Assembler::subq(Address dst, Register src) {
16396 InstructionMark im(this);
16397 emit_prefix_and_int8(get_prefixq(dst, src), 0x29);
16398 emit_operand(src, dst, 0);
16399 }
16400
16401 void Assembler::esubq(Register dst, Address src1, Register src2, bool no_flags) {
16402 InstructionMark im(this);
16403 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16404 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16405 eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16406 emit_int8(0x29);
16407 emit_operand(src2, src1, 0);
16408 }
16409
16410 void Assembler::subq(Register dst, int32_t imm32) {
16411 (void) prefixq_and_encode(dst->encoding());
16412 emit_arith(0x81, 0xE8, dst, imm32);
16413 }
16414
16415 void Assembler::esubq(Register dst, Register src, int32_t imm32, bool no_flags) {
16416 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xE8, no_flags);
16417 }
16418
16419 // Force generation of a 4 byte immediate value even if it fits into 8bit
16420 void Assembler::subq_imm32(Register dst, int32_t imm32) {
16421 (void) prefixq_and_encode(dst->encoding());
16422 emit_arith_imm32(0x81, 0xE8, dst, imm32);
16423 }
16424
16425 void Assembler::esubq_imm32(Register dst, Register src, int32_t imm32, bool no_flags) {
16426 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16427 (void) emit_eevex_prefix_or_demote_ndd(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16428 emit_arith_imm32(0x81, 0xE8, src, imm32);
16429 }
16430
16431 void Assembler::subq(Register dst, Address src) {
16432 InstructionMark im(this);
16433 emit_prefix_and_int8(get_prefixq(src, dst), 0x2B);
16434 emit_operand(dst, src, 0);
16435 }
16436
16437 void Assembler::esubq(Register dst, Register src1, Address src2, bool no_flags) {
16438 InstructionMark im(this);
16439 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x2B, no_flags);
16440 }
16441
16442 void Assembler::subq(Register dst, Register src) {
16443 (void) prefixq_and_encode(dst->encoding(), src->encoding());
16444 emit_arith(0x2B, 0xC0, dst, src);
16445 }
16446
16447 void Assembler::esubq(Register dst, Register src1, Register src2, bool no_flags) {
16448 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16449 // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
16450 // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
16451 (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
16452 emit_arith(0x2B, 0xC0, src1, src2);
16453 }
16454
16455 void Assembler::testq(Address dst, int32_t imm32) {
16456 InstructionMark im(this);
16457 emit_prefix_and_int8(get_prefixq(dst), (unsigned char)0xF7);
16458 emit_operand(as_Register(0), dst, 4);
16459 emit_int32(imm32);
16460 }
16461
16462 void Assembler::testq(Register dst, int32_t imm32) {
16463 // not using emit_arith because test
16464 // doesn't support sign-extension of
16465 // 8bit operands
16466 if (dst == rax) {
16467 prefix(REX_W);
16468 emit_int8((unsigned char)0xA9);
16469 emit_int32(imm32);
16470 } else {
16471 int encode = dst->encoding();
16472 encode = prefixq_and_encode(encode);
16473 emit_int16((unsigned char)0xF7, (0xC0 | encode));
16474 emit_int32(imm32);
16475 }
16476 }
16477
16478 void Assembler::testq(Register dst, Register src) {
16479 (void) prefixq_and_encode(dst->encoding(), src->encoding());
16480 emit_arith(0x85, 0xC0, dst, src);
16481 }
16482
16483 void Assembler::testq(Register dst, Address src) {
16484 InstructionMark im(this);
16485 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x85);
16486 emit_operand(dst, src, 0);
16487 }
16488
16489 void Assembler::xaddq(Address dst, Register src) {
16490 InstructionMark im(this);
16491 int prefix = get_prefixq(dst, src, true /* is_map1 */);
16492 emit_prefix_and_int8(prefix, (unsigned char)0xC1);
16493 emit_operand(src, dst, 0);
16494 }
16495
16496 void Assembler::xchgq(Register dst, Address src) {
16497 InstructionMark im(this);
16498 emit_prefix_and_int8(get_prefixq(src, dst), (unsigned char)0x87);
16499 emit_operand(dst, src, 0);
16500 }
16501
16502 void Assembler::xchgq(Register dst, Register src) {
16503 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
16504 emit_int16((unsigned char)0x87, (0xc0 | encode));
16505 }
16506
16507 void Assembler::xorq(Register dst, Register src) {
16508 (void) prefixq_and_encode(dst->encoding(), src->encoding());
16509 emit_arith(0x33, 0xC0, dst, src);
16510 }
16511
16512 void Assembler::exorq(Register dst, Register src1, Register src2, bool no_flags) {
16513 emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
16514 }
16515
16516 void Assembler::xorq(Register dst, Address src) {
16517 InstructionMark im(this);
16518 emit_prefix_and_int8(get_prefixq(src, dst), 0x33);
16519 emit_operand(dst, src, 0);
16520 }
16521
16522 void Assembler::exorq(Register dst, Register src1, Address src2, bool no_flags) {
16523 InstructionMark im(this);
16524 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, no_flags);
16525 }
16526
16527 void Assembler::xorq(Register dst, int32_t imm32) {
16528 (void) prefixq_and_encode(dst->encoding());
16529 emit_arith(0x81, 0xF0, dst, imm32);
16530 }
16531
16532 void Assembler::exorq(Register dst, Register src, int32_t imm32, bool no_flags) {
16533 emit_eevex_prefix_or_demote_arith_ndd(dst, src, imm32, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x81, 0xF0, no_flags);
16534 }
16535
16536 void Assembler::xorq(Address dst, int32_t imm32) {
16537 InstructionMark im(this);
16538 prefixq(dst);
16539 emit_arith_operand(0x81, as_Register(6), dst, imm32);
16540 }
16541
16542 void Assembler::exorq(Register dst, Address src, int32_t imm32, bool no_flags) {
16543 InstructionMark im(this);
16544 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16545 attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
16546 eevex_prefix_ndd(src, dst->encoding(), 0, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
16547 emit_arith_operand(0x81, as_Register(6), src, imm32);
16548 }
16549
16550 void Assembler::xorq(Address dst, Register src) {
16551 InstructionMark im(this);
16552 emit_prefix_and_int8(get_prefixq(dst, src), 0x31);
16553 emit_operand(src, dst, 0);
16554 }
16555
16556 void Assembler::esetzucc(Condition cc, Register dst) {
16557 assert(VM_Version::supports_apx_f(), "");
16558 assert(0 <= cc && cc < 16, "illegal cc");
16559 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
16560 // Encoding Format : eevex_prefix (4 bytes) | opcode_cc | modrm
16561 int encode = emit_eevex_prefix_ndd(dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3C /* MAP4 */, &attributes); // demotion disabled
16562 emit_opcode_prefix_and_encoding((0x40 | cc), 0xC0, encode);
16563 }
16564
16565 void Assembler::exorq(Register dst, Address src1, Register src2, bool no_flags) {
16566 InstructionMark im(this);
16567 emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
16568 }
16569
16570 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
16571 if (VM_Version::supports_evex()) {
16572 _tuple_type = tuple_type;
16573 _input_size_in_bits = input_size_in_bits;
16574 }
16575 }
16576
16577 void Assembler::evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16578 assert(VM_Version::supports_avx512_vbmi() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16580 attributes.set_is_evex_instruction();
16581 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16582 emit_int16(0x75, (0xC0 | encode));
16583 }
16584
16585 void Assembler::evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16586 assert(VM_Version::supports_avx512bw() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16587 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16588 attributes.set_is_evex_instruction();
16589 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16590 emit_int16(0x75, (0xC0 | encode));
16591 }
16592
16593 void Assembler::evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16594 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16595 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16596 attributes.set_is_evex_instruction();
16597 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16598 emit_int16(0x76, (0xC0 | encode));
16599 }
16600
16601 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16602 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16603 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16604 attributes.set_is_evex_instruction();
16605 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16606 emit_int16(0x76, (0xC0 | encode));
16607 }
16608
16609 void Assembler::evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16610 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16611 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16612 attributes.set_is_evex_instruction();
16613 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16614 emit_int16(0x77, (0xC0 | encode));
16615 }
16616
16617 void Assembler::evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16618 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16619 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16620 attributes.set_is_evex_instruction();
16621 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16622 emit_int16(0x77, (0xC0 | encode));
16623 }
16624
16625 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16626 assert(VM_Version::supports_avx512_vbmi(), "");
16627 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16628 attributes.set_is_evex_instruction();
16629 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16630 emit_int16(0x7D, (0xC0 | encode));
16631 }
16632
16633 void Assembler::evpermt2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16634 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx512vlbw() : VM_Version::supports_avx512bw(), "");
16635 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16636 attributes.set_is_evex_instruction();
16637 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16638 emit_int16(0x7D, (0xC0 | encode));
16639 }
16640
16641 void Assembler::evpermt2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16642 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16643 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16644 attributes.set_is_evex_instruction();
16645 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16646 emit_int16(0x7E, (0xC0 | encode));
16647 }
16648
16649 void Assembler::evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16650 assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
16651 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16652 attributes.set_is_evex_instruction();
16653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
16654 emit_int16(0x7E, (0xC0 | encode));
16655 }
16656
16657 void Assembler::evaddph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16658 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16659 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16660 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16661 attributes.set_is_evex_instruction();
16662 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16663 emit_int16(0x58, (0xC0 | encode));
16664 }
16665
16666 void Assembler::evaddph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16667 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16668 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16669 InstructionMark im(this);
16670 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16671 attributes.set_is_evex_instruction();
16672 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16673 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16674 emit_int8(0x58);
16675 emit_operand(dst, src, 0);
16676 }
16677
16678 void Assembler::evsubph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16679 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16680 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16682 attributes.set_is_evex_instruction();
16683 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16684 emit_int16(0x5C, (0xC0 | encode));
16685 }
16686
16687 void Assembler::evsubph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16688 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16689 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16690 InstructionMark im(this);
16691 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16692 attributes.set_is_evex_instruction();
16693 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16694 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16695 emit_int8(0x5C);
16696 emit_operand(dst, src, 0);
16697 }
16698
16699 void Assembler::evmulph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16700 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16701 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16702 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16703 attributes.set_is_evex_instruction();
16704 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16705 emit_int16(0x59, (0xC0 | encode));
16706 }
16707
16708 void Assembler::evmulph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16709 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16710 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16711 InstructionMark im(this);
16712 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16713 attributes.set_is_evex_instruction();
16714 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16715 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16716 emit_int8(0x59);
16717 emit_operand(dst, src, 0);
16718 }
16719
16720 void Assembler::evminph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16721 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16722 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16723 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16724 attributes.set_is_evex_instruction();
16725 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16726 emit_int16(0x5D, (0xC0 | encode));
16727 }
16728
16729 void Assembler::evminph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16730 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16731 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16732 InstructionMark im(this);
16733 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16734 attributes.set_is_evex_instruction();
16735 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16736 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16737 emit_int8(0x5D);
16738 emit_operand(dst, src, 0);
16739 }
16740
16741 void Assembler::evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len) {
16742 assert(VM_Version::supports_avx10_2(), "");
16743 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
16744 attributes.set_is_evex_instruction();
16745 attributes.set_embedded_opmask_register_specifier(mask);
16746 if (merge) {
16747 attributes.reset_is_clear_context();
16748 }
16749 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
16750 emit_int24(0x52, (0xC0 | encode), imm8);
16751 }
16752
16753 void Assembler::evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len) {
16754 assert(VM_Version::supports_avx10_2(), "");
16755 InstructionMark im(this);
16756 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
16757 attributes.set_is_evex_instruction();
16758 attributes.set_embedded_opmask_register_specifier(mask);
16759 if (merge) {
16760 attributes.reset_is_clear_context();
16761 }
16762 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16763 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
16764 emit_int8(0x52);
16765 emit_operand(dst, src, 0);
16766 emit_int8(imm8);
16767 }
16768
16769 void Assembler::evmaxph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16770 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16771 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16772 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16773 attributes.set_is_evex_instruction();
16774 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16775 emit_int16(0x5F, (0xC0 | encode));
16776 }
16777
16778 void Assembler::evmaxph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16779 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16780 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16781 InstructionMark im(this);
16782 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16783 attributes.set_is_evex_instruction();
16784 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16785 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16786 emit_int8(0x5F);
16787 emit_operand(dst, src, 0);
16788 }
16789
16790 void Assembler::evdivph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16791 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16792 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16793 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16794 attributes.set_is_evex_instruction();
16795 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16796 emit_int16(0x5E, (0xC0 | encode));
16797 }
16798
16799 void Assembler::evdivph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16800 assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
16801 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16802 InstructionMark im(this);
16803 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16804 attributes.set_is_evex_instruction();
16805 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16806 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16807 emit_int8(0x5E);
16808 emit_operand(dst, src, 0);
16809 }
16810
16811 void Assembler::evsqrtph(XMMRegister dst, XMMRegister src, int vector_len) {
16812 assert(VM_Version::supports_avx512_fp16(), "");
16813 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16814 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16815 attributes.set_is_evex_instruction();
16816 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16817 emit_int16(0x51, (0xC0 | encode));
16818 }
16819
16820 void Assembler::evsqrtph(XMMRegister dst, Address src, int vector_len) {
16821 assert(VM_Version::supports_avx512_fp16(), "");
16822 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16823 InstructionMark im(this);
16824 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16825 attributes.set_is_evex_instruction();
16826 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16827 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
16828 emit_int8(0x51);
16829 emit_operand(dst, src, 0);
16830 }
16831
16832 void Assembler::evfmadd132ph(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
16833 assert(VM_Version::supports_avx512_fp16(), "");
16834 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16835 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16836 attributes.set_is_evex_instruction();
16837 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes);
16838 emit_int16(0x98, (0xC0 | encode));
16839 }
16840
16841 void Assembler::evfmadd132ph(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
16842 assert(VM_Version::supports_avx512_fp16(), "");
16843 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
16844 InstructionMark im(this);
16845 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
16846 attributes.set_is_evex_instruction();
16847 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
16848 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP6, &attributes);
16849 emit_int8(0x98);
16850 emit_operand(dst, src, 0);
16851 }
16852