1 /*
2 * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableBarrierSet.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/objectMonitor.hpp"
34 #include "runtime/os.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37 #include "runtime/vm_version.hpp"
38 #include "utilities/macros.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
49 // Implementation of AddressLiteral
50
51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
53 // -----------------Table 4.5 -------------------- //
54 16, 32, 64, // EVEX_FV(0)
55 4, 4, 4, // EVEX_FV(1) - with Evex.b
56 16, 32, 64, // EVEX_FV(2) - with Evex.w
57 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
58 8, 16, 32, // EVEX_HV(0)
59 4, 4, 4, // EVEX_HV(1) - with Evex.b
60 // -----------------Table 4.6 -------------------- //
61 16, 32, 64, // EVEX_FVM(0)
62 1, 1, 1, // EVEX_T1S(0)
63 2, 2, 2, // EVEX_T1S(1)
64 4, 4, 4, // EVEX_T1S(2)
65 8, 8, 8, // EVEX_T1S(3)
66 4, 4, 4, // EVEX_T1F(0)
67 8, 8, 8, // EVEX_T1F(1)
68 8, 8, 8, // EVEX_T2(0)
69 0, 16, 16, // EVEX_T2(1)
70 0, 16, 16, // EVEX_T4(0)
71 0, 0, 32, // EVEX_T4(1)
72 0, 0, 32, // EVEX_T8(0)
73 8, 16, 32, // EVEX_HVM(0)
74 4, 8, 16, // EVEX_QVM(0)
75 2, 4, 8, // EVEX_OVM(0)
76 16, 16, 16, // EVEX_M128(0)
77 8, 32, 64, // EVEX_DUP(0)
78 0, 0, 0 // EVEX_NTUP
79 };
80
81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
82 _is_lval = false;
83 _target = target;
84 switch (rtype) {
85 case relocInfo::oop_type:
86 case relocInfo::metadata_type:
87 // Oops are a special case. Normally they would be their own section
88 // but in cases like icBuffer they are literals in the code stream that
89 // we don't have a section for. We use none so that we get a literal address
90 // which is always patchable.
91 break;
92 case relocInfo::external_word_type:
93 _rspec = external_word_Relocation::spec(target);
94 break;
95 case relocInfo::internal_word_type:
96 _rspec = internal_word_Relocation::spec(target);
97 break;
98 case relocInfo::opt_virtual_call_type:
99 _rspec = opt_virtual_call_Relocation::spec();
100 break;
101 case relocInfo::static_call_type:
102 _rspec = static_call_Relocation::spec();
103 break;
104 case relocInfo::runtime_call_type:
105 _rspec = runtime_call_Relocation::spec();
106 break;
107 case relocInfo::poll_type:
108 case relocInfo::poll_return_type:
109 _rspec = Relocation::spec_simple(rtype);
110 break;
111 case relocInfo::none:
112 break;
113 default:
114 ShouldNotReachHere();
115 break;
116 }
117 }
118
119 // Implementation of Address
120
121 #ifdef _LP64
122
123 Address Address::make_array(ArrayAddress adr) {
124 // Not implementable on 64bit machines
125 // Should have been handled higher up the call chain.
126 ShouldNotReachHere();
127 return Address();
128 }
129
130 // exceedingly dangerous constructor
131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
132 _base = noreg;
133 _index = noreg;
134 _scale = no_scale;
135 _disp = disp;
136 _xmmindex = xnoreg;
137 _isxmmindex = false;
138 switch (rtype) {
139 case relocInfo::external_word_type:
140 _rspec = external_word_Relocation::spec(loc);
141 break;
142 case relocInfo::internal_word_type:
143 _rspec = internal_word_Relocation::spec(loc);
144 break;
145 case relocInfo::runtime_call_type:
146 // HMM
147 _rspec = runtime_call_Relocation::spec();
148 break;
149 case relocInfo::poll_type:
150 case relocInfo::poll_return_type:
151 _rspec = Relocation::spec_simple(rtype);
152 break;
153 case relocInfo::none:
154 break;
155 default:
156 ShouldNotReachHere();
157 }
158 }
159 #else // LP64
160
161 Address Address::make_array(ArrayAddress adr) {
162 AddressLiteral base = adr.base();
163 Address index = adr.index();
164 assert(index._disp == 0, "must not have disp"); // maybe it can?
165 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
166 array._rspec = base._rspec;
167 return array;
168 }
169
170 // exceedingly dangerous constructor
171 Address::Address(address loc, RelocationHolder spec) {
172 _base = noreg;
173 _index = noreg;
174 _scale = no_scale;
175 _disp = (intptr_t) loc;
176 _rspec = spec;
177 _xmmindex = xnoreg;
178 _isxmmindex = false;
179 }
180
181 #endif // _LP64
182
183
184
185 // Convert the raw encoding form into the form expected by the constructor for
186 // Address. An index of 4 (rsp) corresponds to having no index, so convert
187 // that to noreg for the Address constructor.
188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
189 RelocationHolder rspec = RelocationHolder::none;
190 if (disp_reloc != relocInfo::none) {
191 rspec = Relocation::spec_simple(disp_reloc);
192 }
193 bool valid_index = index != rsp->encoding();
194 if (valid_index) {
195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
196 madr._rspec = rspec;
197 return madr;
198 } else {
199 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
200 madr._rspec = rspec;
201 return madr;
202 }
203 }
204
205 // Implementation of Assembler
206
207 int AbstractAssembler::code_fill_byte() {
208 return (u_char)'\xF4'; // hlt
209 }
210
211 void Assembler::init_attributes(void) {
212 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
213 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
214 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
215 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
216 NOT_LP64(_is_managed = false;)
217 _attributes = NULL;
218 }
219
220
221 void Assembler::membar(Membar_mask_bits order_constraint) {
222 // We only have to handle StoreLoad
223 if (order_constraint & StoreLoad) {
224 // All usable chips support "locked" instructions which suffice
225 // as barriers, and are much faster than the alternative of
226 // using cpuid instruction. We use here a locked add [esp-C],0.
227 // This is conveniently otherwise a no-op except for blowing
228 // flags, and introducing a false dependency on target memory
229 // location. We can't do anything with flags, but we can avoid
230 // memory dependencies in the current method by locked-adding
231 // somewhere else on the stack. Doing [esp+C] will collide with
232 // something on stack in current method, hence we go for [esp-C].
233 // It is convenient since it is almost always in data cache, for
234 // any small C. We need to step back from SP to avoid data
235 // dependencies with other things on below SP (callee-saves, for
236 // example). Without a clear way to figure out the minimal safe
237 // distance from SP, it makes sense to step back the complete
238 // cache line, as this will also avoid possible second-order effects
239 // with locked ops against the cache line. Our choice of offset
240 // is bounded by x86 operand encoding, which should stay within
241 // [-128; +127] to have the 8-byte displacement encoding.
242 //
243 // Any change to this code may need to revisit other places in
244 // the code where this idiom is used, in particular the
245 // orderAccess code.
246
247 int offset = -VM_Version::L1_line_size();
248 if (offset < -128) {
249 offset = -128;
250 }
251
252 lock();
253 addl(Address(rsp, offset), 0);// Assert the lock# signal here
254 }
255 }
256
257 // make this go away someday
258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
259 if (rtype == relocInfo::none)
260 emit_int32(data);
261 else
262 emit_data(data, Relocation::spec_simple(rtype), format);
263 }
264
265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
266 assert(imm_operand == 0, "default format must be immediate in this file");
267 assert(inst_mark() != NULL, "must be inside InstructionMark");
268 if (rspec.type() != relocInfo::none) {
269 #ifdef ASSERT
270 check_relocation(rspec, format);
271 #endif
272 // Do not use AbstractAssembler::relocate, which is not intended for
273 // embedded words. Instead, relocate to the enclosing instruction.
274
275 // hack. call32 is too wide for mask so use disp32
276 if (format == call32_operand)
277 code_section()->relocate(inst_mark(), rspec, disp32_operand);
278 else
279 code_section()->relocate(inst_mark(), rspec, format);
280 }
281 emit_int32(data);
282 }
283
284 static int encode(Register r) {
285 int enc = r->encoding();
286 if (enc >= 8) {
287 enc -= 8;
288 }
289 return enc;
290 }
291
292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
293 assert(dst->has_byte_register(), "must have byte register");
294 assert(isByte(op1) && isByte(op2), "wrong opcode");
295 assert(isByte(imm8), "not a byte");
296 assert((op1 & 0x01) == 0, "should be 8bit operation");
297 emit_int24(op1, (op2 | encode(dst)), imm8);
298 }
299
300
301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
302 assert(isByte(op1) && isByte(op2), "wrong opcode");
303 assert(op1 == 0x81, "Unexpected opcode");
304 if (is8bit(imm32)) {
305 emit_int24(op1 | 0x02, // set sign bit
306 op2 | encode(dst),
307 imm32 & 0xFF);
308 } else if (dst == rax) {
309 switch (op2) {
310 case 0xD0: emit_int8(0x15); break; // adc
311 case 0xC0: emit_int8(0x05); break; // add
312 case 0xE0: emit_int8(0x25); break; // and
313 case 0xF8: emit_int8(0x3D); break; // cmp
314 case 0xC8: emit_int8(0x0D); break; // or
315 case 0xD8: emit_int8(0x1D); break; // sbb
316 case 0xE8: emit_int8(0x2D); break; // sub
317 case 0xF0: emit_int8(0x35); break; // xor
318 default: ShouldNotReachHere();
319 }
320 emit_int32(imm32);
321 } else {
322 emit_int16(op1, (op2 | encode(dst)));
323 emit_int32(imm32);
324 }
325 }
326
327 // Force generation of a 4 byte immediate value even if it fits into 8bit
328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
329 assert(isByte(op1) && isByte(op2), "wrong opcode");
330 assert((op1 & 0x01) == 1, "should be 32bit operation");
331 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
332 emit_int16(op1, (op2 | encode(dst)));
333 emit_int32(imm32);
334 }
335
336 // immediate-to-memory forms
337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
338 assert((op1 & 0x01) == 1, "should be 32bit operation");
339 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
340 if (is8bit(imm32)) {
341 emit_int8(op1 | 0x02); // set sign bit
342 emit_operand(rm, adr, 1);
343 emit_int8(imm32 & 0xFF);
344 } else {
345 emit_int8(op1);
346 emit_operand(rm, adr, 4);
347 emit_int32(imm32);
348 }
349 }
350
351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
352 assert(op1 == 0x81, "unexpected opcode");
353 emit_int8(op1);
354 emit_operand(rm, adr, 4);
355 emit_int32(imm32);
356 }
357
358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
359 assert(isByte(op1) && isByte(op2), "wrong opcode");
360 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
361 }
362
363
364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
365 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
366 int mod_idx = 0;
367 // We will test if the displacement fits the compressed format and if so
368 // apply the compression to the displacement iff the result is8bit.
369 if (VM_Version::supports_evex() && is_evex_inst) {
370 switch (cur_tuple_type) {
371 case EVEX_FV:
372 if ((cur_encoding & VEX_W) == VEX_W) {
373 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
374 } else {
375 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
376 }
377 break;
378
379 case EVEX_HV:
380 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
381 break;
382
383 case EVEX_FVM:
384 break;
385
386 case EVEX_T1S:
387 switch (in_size_in_bits) {
388 case EVEX_8bit:
389 break;
390
391 case EVEX_16bit:
392 mod_idx = 1;
393 break;
394
395 case EVEX_32bit:
396 mod_idx = 2;
397 break;
398
399 case EVEX_64bit:
400 mod_idx = 3;
401 break;
402 }
403 break;
404
405 case EVEX_T1F:
406 case EVEX_T2:
407 case EVEX_T4:
408 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
409 break;
410
411 case EVEX_T8:
412 break;
413
414 case EVEX_HVM:
415 break;
416
417 case EVEX_QVM:
418 break;
419
420 case EVEX_OVM:
421 break;
422
423 case EVEX_M128:
424 break;
425
426 case EVEX_DUP:
427 break;
428
429 default:
430 assert(0, "no valid evex tuple_table entry");
431 break;
432 }
433
434 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
435 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
436 if ((disp % disp_factor) == 0) {
437 int new_disp = disp / disp_factor;
438 if ((-0x80 <= new_disp && new_disp < 0x80)) {
439 disp = new_disp;
440 }
441 } else {
442 return false;
443 }
444 }
445 }
446 return (-0x80 <= disp && disp < 0x80);
447 }
448
449
450 bool Assembler::emit_compressed_disp_byte(int &disp) {
451 int mod_idx = 0;
452 // We will test if the displacement fits the compressed format and if so
453 // apply the compression to the displacement iff the result is8bit.
454 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
455 int evex_encoding = _attributes->get_evex_encoding();
456 int tuple_type = _attributes->get_tuple_type();
457 switch (tuple_type) {
458 case EVEX_FV:
459 if ((evex_encoding & VEX_W) == VEX_W) {
460 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
461 } else {
462 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
463 }
464 break;
465
466 case EVEX_HV:
467 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
468 break;
469
470 case EVEX_FVM:
471 break;
472
473 case EVEX_T1S:
474 switch (_attributes->get_input_size()) {
475 case EVEX_8bit:
476 break;
477
478 case EVEX_16bit:
479 mod_idx = 1;
480 break;
481
482 case EVEX_32bit:
483 mod_idx = 2;
484 break;
485
486 case EVEX_64bit:
487 mod_idx = 3;
488 break;
489 }
490 break;
491
492 case EVEX_T1F:
493 case EVEX_T2:
494 case EVEX_T4:
495 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
496 break;
497
498 case EVEX_T8:
499 break;
500
501 case EVEX_HVM:
502 break;
503
504 case EVEX_QVM:
505 break;
506
507 case EVEX_OVM:
508 break;
509
510 case EVEX_M128:
511 break;
512
513 case EVEX_DUP:
514 break;
515
516 default:
517 assert(0, "no valid evex tuple_table entry");
518 break;
519 }
520
521 int vector_len = _attributes->get_vector_len();
522 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
523 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
524 if ((disp % disp_factor) == 0) {
525 int new_disp = disp / disp_factor;
526 if (is8bit(new_disp)) {
527 disp = new_disp;
528 }
529 } else {
530 return false;
531 }
532 }
533 }
534 return is8bit(disp);
535 }
536
537 static bool is_valid_encoding(int reg_enc) {
538 return reg_enc >= 0;
539 }
540
541 static int raw_encode(Register reg) {
542 assert(reg == noreg || reg->is_valid(), "sanity");
543 int reg_enc = reg->raw_encoding();
544 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
545 return reg_enc;
546 }
547
548 static int raw_encode(XMMRegister xmmreg) {
549 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
550 int xmmreg_enc = xmmreg->raw_encoding();
551 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
552 return xmmreg_enc;
553 }
554
555 static int raw_encode(KRegister kreg) {
556 assert(kreg == knoreg || kreg->is_valid(), "sanity");
557 int kreg_enc = kreg->raw_encoding();
558 assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
559 return kreg_enc;
560 }
561
562 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
563 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
564 }
565
566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
567 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
568 }
569
570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
571 assert((mod & 3) != 0b11, "forbidden");
572 int modrm = modrm_encoding(mod, dst_enc, src_enc);
573 emit_int8(modrm);
574 }
575
576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
577 int disp) {
578 int modrm = modrm_encoding(mod, dst_enc, src_enc);
579 emit_int16(modrm, disp & 0xFF);
580 }
581
582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
583 Address::ScaleFactor scale, int index_enc, int base_enc) {
584 int modrm = modrm_encoding(mod, dst_enc, src_enc);
585 int sib = sib_encoding(scale, index_enc, base_enc);
586 emit_int16(modrm, sib);
587 }
588
589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
590 Address::ScaleFactor scale, int index_enc, int base_enc,
591 int disp) {
592 int modrm = modrm_encoding(mod, dst_enc, src_enc);
593 int sib = sib_encoding(scale, index_enc, base_enc);
594 emit_int24(modrm, sib, disp & 0xFF);
595 }
596
597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
598 Address::ScaleFactor scale, int disp,
599 RelocationHolder const& rspec,
600 int post_addr_length) {
601 bool no_relocation = (rspec.type() == relocInfo::none);
602
603 if (is_valid_encoding(base_enc)) {
604 if (is_valid_encoding(index_enc)) {
605 assert(scale != Address::no_scale, "inconsistent address");
606 // [base + index*scale + disp]
607 if (disp == 0 && no_relocation &&
608 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
609 // [base + index*scale]
610 // [00 reg 100][ss index base]
611 emit_modrm_sib(0b00, reg_enc, 0b100,
612 scale, index_enc, base_enc);
613 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
614 // [base + index*scale + imm8]
615 // [01 reg 100][ss index base] imm8
616 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
617 scale, index_enc, base_enc,
618 disp);
619 } else {
620 // [base + index*scale + disp32]
621 // [10 reg 100][ss index base] disp32
622 emit_modrm_sib(0b10, reg_enc, 0b100,
623 scale, index_enc, base_enc);
624 emit_data(disp, rspec, disp32_operand);
625 }
626 } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
627 // [rsp + disp]
628 if (disp == 0 && no_relocation) {
629 // [rsp]
630 // [00 reg 100][00 100 100]
631 emit_modrm_sib(0b00, reg_enc, 0b100,
632 Address::times_1, 0b100, 0b100);
633 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
634 // [rsp + imm8]
635 // [01 reg 100][00 100 100] disp8
636 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
637 Address::times_1, 0b100, 0b100,
638 disp);
639 } else {
640 // [rsp + imm32]
641 // [10 reg 100][00 100 100] disp32
642 emit_modrm_sib(0b10, reg_enc, 0b100,
643 Address::times_1, 0b100, 0b100);
644 emit_data(disp, rspec, disp32_operand);
645 }
646 } else {
647 // [base + disp]
648 assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
649 if (disp == 0 && no_relocation &&
650 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
651 // [base]
652 // [00 reg base]
653 emit_modrm(0, reg_enc, base_enc);
654 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
655 // [base + disp8]
656 // [01 reg base] disp8
657 emit_modrm_disp8(0b01, reg_enc, base_enc,
658 disp);
659 } else {
660 // [base + disp32]
661 // [10 reg base] disp32
662 emit_modrm(0b10, reg_enc, base_enc);
663 emit_data(disp, rspec, disp32_operand);
664 }
665 }
666 } else {
667 if (is_valid_encoding(index_enc)) {
668 assert(scale != Address::no_scale, "inconsistent address");
669 // base == noreg
670 // [index*scale + disp]
671 // [00 reg 100][ss index 101] disp32
672 emit_modrm_sib(0b00, reg_enc, 0b100,
673 scale, index_enc, 0b101 /* no base */);
674 emit_data(disp, rspec, disp32_operand);
675 } else if (!no_relocation) {
676 // base == noreg, index == noreg
677 // [disp] (64bit) RIP-RELATIVE (32bit) abs
678 // [00 reg 101] disp32
679
680 emit_modrm(0b00, reg_enc, 0b101 /* no base */);
681 // Note that the RIP-rel. correction applies to the generated
682 // disp field, but _not_ to the target address in the rspec.
683
684 // disp was created by converting the target address minus the pc
685 // at the start of the instruction. That needs more correction here.
686 // intptr_t disp = target - next_ip;
687 assert(inst_mark() != NULL, "must be inside InstructionMark");
688 address next_ip = pc() + sizeof(int32_t) + post_addr_length;
689 int64_t adjusted = disp;
690 // Do rip-rel adjustment for 64bit
691 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
692 assert(is_simm32(adjusted),
693 "must be 32bit offset (RIP relative address)");
694 emit_data((int32_t) adjusted, rspec, disp32_operand);
695
696 } else {
697 // base == noreg, index == noreg, no_relocation == true
698 // 32bit never did this, did everything as the rip-rel/disp code above
699 // [disp] ABSOLUTE
700 // [00 reg 100][00 100 101] disp32
701 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
702 Address::times_1, 0b100, 0b101);
703 emit_data(disp, rspec, disp32_operand);
704 }
705 }
706 }
707
708 void Assembler::emit_operand(Register reg, Register base, Register index,
709 Address::ScaleFactor scale, int disp,
710 RelocationHolder const& rspec,
711 int post_addr_length) {
712 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
713 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
714 scale, disp, rspec, post_addr_length);
715
716 }
717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
718 Address::ScaleFactor scale, int disp,
719 RelocationHolder const& rspec,
720 int post_addr_length) {
721 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
722 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
723 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
724 scale, disp, rspec, post_addr_length);
725 }
726
727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
728 Address::ScaleFactor scale, int disp,
729 RelocationHolder const& rspec,
730 int post_addr_length) {
731 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
732 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
733 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
734 scale, disp, rspec, post_addr_length);
735 }
736
737 void Assembler::emit_operand(KRegister kreg, Address adr,
738 int post_addr_length) {
739 emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
740 adr._rspec,
741 post_addr_length);
742 }
743
744 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
745 Address::ScaleFactor scale, int disp,
746 RelocationHolder const& rspec,
747 int post_addr_length) {
748 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
749 emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
750 scale, disp, rspec, post_addr_length);
751 }
752
753 // Secret local extension to Assembler::WhichOperand:
754 #define end_pc_operand (_WhichOperand_limit)
755
756 address Assembler::locate_operand(address inst, WhichOperand which) {
757 // Decode the given instruction, and return the address of
758 // an embedded 32-bit operand word.
759
760 // If "which" is disp32_operand, selects the displacement portion
761 // of an effective address specifier.
762 // If "which" is imm64_operand, selects the trailing immediate constant.
763 // If "which" is call32_operand, selects the displacement of a call or jump.
764 // Caller is responsible for ensuring that there is such an operand,
765 // and that it is 32/64 bits wide.
766
767 // If "which" is end_pc_operand, find the end of the instruction.
768
769 address ip = inst;
770 bool is_64bit = false;
771
772 debug_only(bool has_disp32 = false);
773 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
774
775 again_after_prefix:
776 switch (0xFF & *ip++) {
777
778 // These convenience macros generate groups of "case" labels for the switch.
779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
781 case (x)+4: case (x)+5: case (x)+6: case (x)+7
782 #define REP16(x) REP8((x)+0): \
783 case REP8((x)+8)
784
785 case CS_segment:
786 case SS_segment:
787 case DS_segment:
788 case ES_segment:
789 case FS_segment:
790 case GS_segment:
791 // Seems dubious
792 LP64_ONLY(assert(false, "shouldn't have that prefix"));
793 assert(ip == inst+1, "only one prefix allowed");
794 goto again_after_prefix;
795
796 case 0x67:
797 case REX:
798 case REX_B:
799 case REX_X:
800 case REX_XB:
801 case REX_R:
802 case REX_RB:
803 case REX_RX:
804 case REX_RXB:
805 NOT_LP64(assert(false, "64bit prefixes"));
806 goto again_after_prefix;
807
808 case REX_W:
809 case REX_WB:
810 case REX_WX:
811 case REX_WXB:
812 case REX_WR:
813 case REX_WRB:
814 case REX_WRX:
815 case REX_WRXB:
816 NOT_LP64(assert(false, "64bit prefixes"));
817 is_64bit = true;
818 goto again_after_prefix;
819
820 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
821 case 0x88: // movb a, r
822 case 0x89: // movl a, r
823 case 0x8A: // movb r, a
824 case 0x8B: // movl r, a
825 case 0x8F: // popl a
826 debug_only(has_disp32 = true);
827 break;
828
829 case 0x68: // pushq #32
830 if (which == end_pc_operand) {
831 return ip + 4;
832 }
833 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
834 return ip; // not produced by emit_operand
835
836 case 0x66: // movw ... (size prefix)
837 again_after_size_prefix2:
838 switch (0xFF & *ip++) {
839 case REX:
840 case REX_B:
841 case REX_X:
842 case REX_XB:
843 case REX_R:
844 case REX_RB:
845 case REX_RX:
846 case REX_RXB:
847 case REX_W:
848 case REX_WB:
849 case REX_WX:
850 case REX_WXB:
851 case REX_WR:
852 case REX_WRB:
853 case REX_WRX:
854 case REX_WRXB:
855 NOT_LP64(assert(false, "64bit prefix found"));
856 goto again_after_size_prefix2;
857 case 0x8B: // movw r, a
858 case 0x89: // movw a, r
859 debug_only(has_disp32 = true);
860 break;
861 case 0xC7: // movw a, #16
862 debug_only(has_disp32 = true);
863 tail_size = 2; // the imm16
864 break;
865 case 0x0F: // several SSE/SSE2 variants
866 ip--; // reparse the 0x0F
867 goto again_after_prefix;
868 default:
869 ShouldNotReachHere();
870 }
871 break;
872
873 case REP8(0xB8): // movl/q r, #32/#64(oop?)
874 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
875 // these asserts are somewhat nonsensical
876 #ifndef _LP64
877 assert(which == imm_operand || which == disp32_operand,
878 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
879 #else
880 assert((which == call32_operand || which == imm_operand) && is_64bit ||
881 which == narrow_oop_operand && !is_64bit,
882 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
883 #endif // _LP64
884 return ip;
885
886 case 0x69: // imul r, a, #32
887 case 0xC7: // movl a, #32(oop?)
888 tail_size = 4;
889 debug_only(has_disp32 = true); // has both kinds of operands!
890 break;
891
892 case 0x0F: // movx..., etc.
893 switch (0xFF & *ip++) {
894 case 0x3A: // pcmpestri
895 tail_size = 1;
896 case 0x38: // ptest, pmovzxbw
897 ip++; // skip opcode
898 debug_only(has_disp32 = true); // has both kinds of operands!
899 break;
900
901 case 0x70: // pshufd r, r/a, #8
902 debug_only(has_disp32 = true); // has both kinds of operands!
903 case 0x73: // psrldq r, #8
904 tail_size = 1;
905 break;
906
907 case 0x10: // movups
908 case 0x11: // movups
909 case 0x12: // movlps
910 case 0x28: // movaps
911 case 0x2E: // ucomiss
912 case 0x2F: // comiss
913 case 0x54: // andps
914 case 0x55: // andnps
915 case 0x56: // orps
916 case 0x57: // xorps
917 case 0x58: // addpd
918 case 0x59: // mulpd
919 case 0x6E: // movd
920 case 0x7E: // movd
921 case 0x6F: // movdq
922 case 0x7F: // movdq
923 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
924 case 0xFE: // paddd
925 debug_only(has_disp32 = true);
926 break;
927
928 case 0xAD: // shrd r, a, %cl
929 case 0xAF: // imul r, a
930 case 0xBE: // movsbl r, a (movsxb)
931 case 0xBF: // movswl r, a (movsxw)
932 case 0xB6: // movzbl r, a (movzxb)
933 case 0xB7: // movzwl r, a (movzxw)
934 case REP16(0x40): // cmovl cc, r, a
935 case 0xB0: // cmpxchgb
936 case 0xB1: // cmpxchg
937 case 0xC1: // xaddl
938 case 0xC7: // cmpxchg8
939 case REP16(0x90): // setcc a
940 debug_only(has_disp32 = true);
941 // fall out of the switch to decode the address
942 break;
943
944 case 0xC4: // pinsrw r, a, #8
945 debug_only(has_disp32 = true);
946 case 0xC5: // pextrw r, r, #8
947 tail_size = 1; // the imm8
948 break;
949
950 case 0xAC: // shrd r, a, #8
951 debug_only(has_disp32 = true);
952 tail_size = 1; // the imm8
953 break;
954
955 case REP16(0x80): // jcc rdisp32
956 if (which == end_pc_operand) return ip + 4;
957 assert(which == call32_operand, "jcc has no disp32 or imm");
958 return ip;
959 default:
960 ShouldNotReachHere();
961 }
962 break;
963
964 case 0x81: // addl a, #32; addl r, #32
965 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
966 // on 32bit in the case of cmpl, the imm might be an oop
967 tail_size = 4;
968 debug_only(has_disp32 = true); // has both kinds of operands!
969 break;
970
971 case 0x83: // addl a, #8; addl r, #8
972 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
973 debug_only(has_disp32 = true); // has both kinds of operands!
974 tail_size = 1;
975 break;
976
977 case 0x15: // adc rax, #32
978 case 0x05: // add rax, #32
979 case 0x25: // and rax, #32
980 case 0x3D: // cmp rax, #32
981 case 0x0D: // or rax, #32
982 case 0x1D: // sbb rax, #32
983 case 0x2D: // sub rax, #32
984 case 0x35: // xor rax, #32
985 return which == end_pc_operand ? ip + 4 : ip;
986
987 case 0x9B:
988 switch (0xFF & *ip++) {
989 case 0xD9: // fnstcw a
990 debug_only(has_disp32 = true);
991 break;
992 default:
993 ShouldNotReachHere();
994 }
995 break;
996
997 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
998 case REP4(0x10): // adc...
999 case REP4(0x20): // and...
1000 case REP4(0x30): // xor...
1001 case REP4(0x08): // or...
1002 case REP4(0x18): // sbb...
1003 case REP4(0x28): // sub...
1004 case 0xF7: // mull a
1005 case 0x8D: // lea r, a
1006 case 0x87: // xchg r, a
1007 case REP4(0x38): // cmp...
1008 case 0x85: // test r, a
1009 debug_only(has_disp32 = true); // has both kinds of operands!
1010 break;
1011
1012 case 0xA8: // testb rax, #8
1013 return which == end_pc_operand ? ip + 1 : ip;
1014 case 0xA9: // testl/testq rax, #32
1015 return which == end_pc_operand ? ip + 4 : ip;
1016
1017 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
1018 case 0xC6: // movb a, #8
1019 case 0x80: // cmpb a, #8
1020 case 0x6B: // imul r, a, #8
1021 debug_only(has_disp32 = true); // has both kinds of operands!
1022 tail_size = 1; // the imm8
1023 break;
1024
1025 case 0xC4: // VEX_3bytes
1026 case 0xC5: // VEX_2bytes
1027 assert((UseAVX > 0), "shouldn't have VEX prefix");
1028 assert(ip == inst+1, "no prefixes allowed");
1029 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
1030 // but they have prefix 0x0F and processed when 0x0F processed above.
1031 //
1032 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
1033 // instructions (these instructions are not supported in 64-bit mode).
1034 // To distinguish them bits [7:6] are set in the VEX second byte since
1035 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
1036 // those VEX bits REX and vvvv bits are inverted.
1037 //
1038 // Fortunately C2 doesn't generate these instructions so we don't need
1039 // to check for them in product version.
1040
1041 // Check second byte
1042 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
1043
1044 int vex_opcode;
1045 // First byte
1046 if ((0xFF & *inst) == VEX_3bytes) {
1047 vex_opcode = VEX_OPCODE_MASK & *ip;
1048 ip++; // third byte
1049 is_64bit = ((VEX_W & *ip) == VEX_W);
1050 } else {
1051 vex_opcode = VEX_OPCODE_0F;
1052 }
1053 ip++; // opcode
1054 // To find the end of instruction (which == end_pc_operand).
1055 switch (vex_opcode) {
1056 case VEX_OPCODE_0F:
1057 switch (0xFF & *ip) {
1058 case 0x70: // pshufd r, r/a, #8
1059 case 0x71: // ps[rl|ra|ll]w r, #8
1060 case 0x72: // ps[rl|ra|ll]d r, #8
1061 case 0x73: // ps[rl|ra|ll]q r, #8
1062 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
1063 case 0xC4: // pinsrw r, r, r/a, #8
1064 case 0xC5: // pextrw r/a, r, #8
1065 case 0xC6: // shufp[s|d] r, r, r/a, #8
1066 tail_size = 1; // the imm8
1067 break;
1068 }
1069 break;
1070 case VEX_OPCODE_0F_3A:
1071 tail_size = 1;
1072 break;
1073 }
1074 ip++; // skip opcode
1075 debug_only(has_disp32 = true); // has both kinds of operands!
1076 break;
1077
1078 case 0x62: // EVEX_4bytes
1079 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
1080 assert(ip == inst+1, "no prefixes allowed");
1081 // no EVEX collisions, all instructions that have 0x62 opcodes
1082 // have EVEX versions and are subopcodes of 0x66
1083 ip++; // skip P0 and examine W in P1
1084 is_64bit = ((VEX_W & *ip) == VEX_W);
1085 ip++; // move to P2
1086 ip++; // skip P2, move to opcode
1087 // To find the end of instruction (which == end_pc_operand).
1088 switch (0xFF & *ip) {
1089 case 0x22: // pinsrd r, r/a, #8
1090 case 0x61: // pcmpestri r, r/a, #8
1091 case 0x70: // pshufd r, r/a, #8
1092 case 0x73: // psrldq r, #8
1093 case 0x1f: // evpcmpd/evpcmpq
1094 case 0x3f: // evpcmpb/evpcmpw
1095 tail_size = 1; // the imm8
1096 break;
1097 default:
1098 break;
1099 }
1100 ip++; // skip opcode
1101 debug_only(has_disp32 = true); // has both kinds of operands!
1102 break;
1103
1104 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
1105 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
1106 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
1107 case 0xDD: // fld_d a; fst_d a; fstp_d a
1108 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
1109 case 0xDF: // fild_d a; fistp_d a
1110 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
1111 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
1112 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
1113 debug_only(has_disp32 = true);
1114 break;
1115
1116 case 0xE8: // call rdisp32
1117 case 0xE9: // jmp rdisp32
1118 if (which == end_pc_operand) return ip + 4;
1119 assert(which == call32_operand, "call has no disp32 or imm");
1120 return ip;
1121
1122 case 0xF0: // Lock
1123 goto again_after_prefix;
1124
1125 case 0xF3: // For SSE
1126 case 0xF2: // For SSE2
1127 switch (0xFF & *ip++) {
1128 case REX:
1129 case REX_B:
1130 case REX_X:
1131 case REX_XB:
1132 case REX_R:
1133 case REX_RB:
1134 case REX_RX:
1135 case REX_RXB:
1136 case REX_W:
1137 case REX_WB:
1138 case REX_WX:
1139 case REX_WXB:
1140 case REX_WR:
1141 case REX_WRB:
1142 case REX_WRX:
1143 case REX_WRXB:
1144 NOT_LP64(assert(false, "found 64bit prefix"));
1145 ip++;
1146 default:
1147 ip++;
1148 }
1149 debug_only(has_disp32 = true); // has both kinds of operands!
1150 break;
1151
1152 default:
1153 ShouldNotReachHere();
1154
1155 #undef REP8
1156 #undef REP16
1157 }
1158
1159 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1160 #ifdef _LP64
1161 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1162 #else
1163 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1164 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1165 #endif // LP64
1166 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1167
1168 // parse the output of emit_operand
1169 int op2 = 0xFF & *ip++;
1170 int base = op2 & 0x07;
1171 int op3 = -1;
1172 const int b100 = 4;
1173 const int b101 = 5;
1174 if (base == b100 && (op2 >> 6) != 3) {
1175 op3 = 0xFF & *ip++;
1176 base = op3 & 0x07; // refetch the base
1177 }
1178 // now ip points at the disp (if any)
1179
1180 switch (op2 >> 6) {
1181 case 0:
1182 // [00 reg 100][ss index base]
1183 // [00 reg 100][00 100 esp]
1184 // [00 reg base]
1185 // [00 reg 100][ss index 101][disp32]
1186 // [00 reg 101] [disp32]
1187
1188 if (base == b101) {
1189 if (which == disp32_operand)
1190 return ip; // caller wants the disp32
1191 ip += 4; // skip the disp32
1192 }
1193 break;
1194
1195 case 1:
1196 // [01 reg 100][ss index base][disp8]
1197 // [01 reg 100][00 100 esp][disp8]
1198 // [01 reg base] [disp8]
1199 ip += 1; // skip the disp8
1200 break;
1201
1202 case 2:
1203 // [10 reg 100][ss index base][disp32]
1204 // [10 reg 100][00 100 esp][disp32]
1205 // [10 reg base] [disp32]
1206 if (which == disp32_operand)
1207 return ip; // caller wants the disp32
1208 ip += 4; // skip the disp32
1209 break;
1210
1211 case 3:
1212 // [11 reg base] (not a memory addressing mode)
1213 break;
1214 }
1215
1216 if (which == end_pc_operand) {
1217 return ip + tail_size;
1218 }
1219
1220 #ifdef _LP64
1221 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1222 #else
1223 assert(which == imm_operand, "instruction has only an imm field");
1224 #endif // LP64
1225 return ip;
1226 }
1227
1228 address Assembler::locate_next_instruction(address inst) {
1229 // Secretly share code with locate_operand:
1230 return locate_operand(inst, end_pc_operand);
1231 }
1232
1233
1234 #ifdef ASSERT
1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1236 address inst = inst_mark();
1237 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1238 address opnd;
1239
1240 Relocation* r = rspec.reloc();
1241 if (r->type() == relocInfo::none) {
1242 return;
1243 } else if (r->is_call() || format == call32_operand) {
1244 // assert(format == imm32_operand, "cannot specify a nonzero format");
1245 opnd = locate_operand(inst, call32_operand);
1246 } else if (r->is_data()) {
1247 assert(format == imm_operand || format == disp32_operand
1248 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1249 opnd = locate_operand(inst, (WhichOperand)format);
1250 } else {
1251 assert(format == imm_operand, "cannot specify a format");
1252 return;
1253 }
1254 assert(opnd == pc(), "must put operand where relocs can find it");
1255 }
1256 #endif // ASSERT
1257
1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
1259 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1260 }
1261
1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
1263 if (adr.isxmmindex()) {
1264 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
1265 } else {
1266 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1267 }
1268 }
1269
1270 // Now the Assembler instructions (identical for 32/64 bits)
1271
1272 void Assembler::adcl(Address dst, int32_t imm32) {
1273 InstructionMark im(this);
1274 prefix(dst);
1275 emit_arith_operand(0x81, rdx, dst, imm32);
1276 }
1277
1278 void Assembler::adcl(Address dst, Register src) {
1279 InstructionMark im(this);
1280 prefix(dst, src);
1281 emit_int8(0x11);
1282 emit_operand(src, dst, 0);
1283 }
1284
1285 void Assembler::adcl(Register dst, int32_t imm32) {
1286 prefix(dst);
1287 emit_arith(0x81, 0xD0, dst, imm32);
1288 }
1289
1290 void Assembler::adcl(Register dst, Address src) {
1291 InstructionMark im(this);
1292 prefix(src, dst);
1293 emit_int8(0x13);
1294 emit_operand(dst, src, 0);
1295 }
1296
1297 void Assembler::adcl(Register dst, Register src) {
1298 (void) prefix_and_encode(dst->encoding(), src->encoding());
1299 emit_arith(0x13, 0xC0, dst, src);
1300 }
1301
1302 void Assembler::addl(Address dst, int32_t imm32) {
1303 InstructionMark im(this);
1304 prefix(dst);
1305 emit_arith_operand(0x81, rax, dst, imm32);
1306 }
1307
1308 void Assembler::addb(Address dst, int imm8) {
1309 InstructionMark im(this);
1310 prefix(dst);
1311 emit_int8((unsigned char)0x80);
1312 emit_operand(rax, dst, 1);
1313 emit_int8(imm8);
1314 }
1315
1316 void Assembler::addw(Register dst, Register src) {
1317 emit_int8(0x66);
1318 (void)prefix_and_encode(dst->encoding(), src->encoding());
1319 emit_arith(0x03, 0xC0, dst, src);
1320 }
1321
1322 void Assembler::addw(Address dst, int imm16) {
1323 InstructionMark im(this);
1324 emit_int8(0x66);
1325 prefix(dst);
1326 emit_int8((unsigned char)0x81);
1327 emit_operand(rax, dst, 2);
1328 emit_int16(imm16);
1329 }
1330
1331 void Assembler::addl(Address dst, Register src) {
1332 InstructionMark im(this);
1333 prefix(dst, src);
1334 emit_int8(0x01);
1335 emit_operand(src, dst, 0);
1336 }
1337
1338 void Assembler::addl(Register dst, int32_t imm32) {
1339 prefix(dst);
1340 emit_arith(0x81, 0xC0, dst, imm32);
1341 }
1342
1343 void Assembler::addl(Register dst, Address src) {
1344 InstructionMark im(this);
1345 prefix(src, dst);
1346 emit_int8(0x03);
1347 emit_operand(dst, src, 0);
1348 }
1349
1350 void Assembler::addl(Register dst, Register src) {
1351 (void) prefix_and_encode(dst->encoding(), src->encoding());
1352 emit_arith(0x03, 0xC0, dst, src);
1353 }
1354
1355 void Assembler::addr_nop_4() {
1356 assert(UseAddressNop, "no CPU support");
1357 // 4 bytes: NOP DWORD PTR [EAX+0]
1358 emit_int32(0x0F,
1359 0x1F,
1360 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1361 0); // 8-bits offset (1 byte)
1362 }
1363
1364 void Assembler::addr_nop_5() {
1365 assert(UseAddressNop, "no CPU support");
1366 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1367 emit_int32(0x0F,
1368 0x1F,
1369 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1370 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1371 emit_int8(0); // 8-bits offset (1 byte)
1372 }
1373
1374 void Assembler::addr_nop_7() {
1375 assert(UseAddressNop, "no CPU support");
1376 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1377 emit_int24(0x0F,
1378 0x1F,
1379 (unsigned char)0x80);
1380 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1381 emit_int32(0); // 32-bits offset (4 bytes)
1382 }
1383
1384 void Assembler::addr_nop_8() {
1385 assert(UseAddressNop, "no CPU support");
1386 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1387 emit_int32(0x0F,
1388 0x1F,
1389 (unsigned char)0x84,
1390 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1391 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1392 emit_int32(0); // 32-bits offset (4 bytes)
1393 }
1394
1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1396 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1397 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1398 attributes.set_rex_vex_w_reverted();
1399 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1400 emit_int16(0x58, (0xC0 | encode));
1401 }
1402
1403 void Assembler::addsd(XMMRegister dst, Address src) {
1404 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1405 InstructionMark im(this);
1406 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1407 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1408 attributes.set_rex_vex_w_reverted();
1409 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1410 emit_int8(0x58);
1411 emit_operand(dst, src, 0);
1412 }
1413
1414 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1415 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1416 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1417 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1418 emit_int16(0x58, (0xC0 | encode));
1419 }
1420
1421 void Assembler::addss(XMMRegister dst, Address src) {
1422 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1423 InstructionMark im(this);
1424 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1425 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1426 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1427 emit_int8(0x58);
1428 emit_operand(dst, src, 0);
1429 }
1430
1431 void Assembler::aesdec(XMMRegister dst, Address src) {
1432 assert(VM_Version::supports_aes(), "");
1433 InstructionMark im(this);
1434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1435 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1436 emit_int8((unsigned char)0xDE);
1437 emit_operand(dst, src, 0);
1438 }
1439
1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1441 assert(VM_Version::supports_aes(), "");
1442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1443 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1444 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1445 }
1446
1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1448 assert(VM_Version::supports_avx512_vaes(), "");
1449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1450 attributes.set_is_evex_instruction();
1451 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1452 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1453 }
1454
1455
1456 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1457 assert(VM_Version::supports_aes(), "");
1458 InstructionMark im(this);
1459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1460 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1461 emit_int8((unsigned char)0xDF);
1462 emit_operand(dst, src, 0);
1463 }
1464
1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1466 assert(VM_Version::supports_aes(), "");
1467 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1468 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1469 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1470 }
1471
1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1473 assert(VM_Version::supports_avx512_vaes(), "");
1474 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1475 attributes.set_is_evex_instruction();
1476 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1477 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1478 }
1479
1480 void Assembler::aesenc(XMMRegister dst, Address src) {
1481 assert(VM_Version::supports_aes(), "");
1482 InstructionMark im(this);
1483 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1484 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1485 emit_int8((unsigned char)0xDC);
1486 emit_operand(dst, src, 0);
1487 }
1488
1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1490 assert(VM_Version::supports_aes(), "");
1491 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1492 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1493 emit_int16((unsigned char)0xDC, 0xC0 | encode);
1494 }
1495
1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1497 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1498 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1499 attributes.set_is_evex_instruction();
1500 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1501 emit_int16((unsigned char)0xDC, (0xC0 | encode));
1502 }
1503
1504 void Assembler::aesenclast(XMMRegister dst, Address src) {
1505 assert(VM_Version::supports_aes(), "");
1506 InstructionMark im(this);
1507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1508 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1509 emit_int8((unsigned char)0xDD);
1510 emit_operand(dst, src, 0);
1511 }
1512
1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1514 assert(VM_Version::supports_aes(), "");
1515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1517 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1518 }
1519
1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1521 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1522 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1523 attributes.set_is_evex_instruction();
1524 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1525 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1526 }
1527
1528 void Assembler::andb(Address dst, Register src) {
1529 InstructionMark im(this);
1530 prefix(dst, src, true);
1531 emit_int8(0x20);
1532 emit_operand(src, dst, 0);
1533 }
1534
1535 void Assembler::andw(Register dst, Register src) {
1536 (void)prefix_and_encode(dst->encoding(), src->encoding());
1537 emit_arith(0x23, 0xC0, dst, src);
1538 }
1539
1540 void Assembler::andl(Address dst, int32_t imm32) {
1541 InstructionMark im(this);
1542 prefix(dst);
1543 emit_arith_operand(0x81, as_Register(4), dst, imm32);
1544 }
1545
1546 void Assembler::andl(Register dst, int32_t imm32) {
1547 prefix(dst);
1548 emit_arith(0x81, 0xE0, dst, imm32);
1549 }
1550
1551 void Assembler::andl(Address dst, Register src) {
1552 InstructionMark im(this);
1553 prefix(dst, src);
1554 emit_int8(0x21);
1555 emit_operand(src, dst, 0);
1556 }
1557
1558 void Assembler::andl(Register dst, Address src) {
1559 InstructionMark im(this);
1560 prefix(src, dst);
1561 emit_int8(0x23);
1562 emit_operand(dst, src, 0);
1563 }
1564
1565 void Assembler::andl(Register dst, Register src) {
1566 (void) prefix_and_encode(dst->encoding(), src->encoding());
1567 emit_arith(0x23, 0xC0, dst, src);
1568 }
1569
1570 void Assembler::andnl(Register dst, Register src1, Register src2) {
1571 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1572 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1573 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1574 emit_int16((unsigned char)0xF2, (0xC0 | encode));
1575 }
1576
1577 void Assembler::andnl(Register dst, Register src1, Address src2) {
1578 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1579 InstructionMark im(this);
1580 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1581 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1582 emit_int8((unsigned char)0xF2);
1583 emit_operand(dst, src2, 0);
1584 }
1585
1586 void Assembler::bsfl(Register dst, Register src) {
1587 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1588 emit_int24(0x0F,
1589 (unsigned char)0xBC,
1590 0xC0 | encode);
1591 }
1592
1593 void Assembler::bsrl(Register dst, Register src) {
1594 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1595 emit_int24(0x0F,
1596 (unsigned char)0xBD,
1597 0xC0 | encode);
1598 }
1599
1600 void Assembler::bswapl(Register reg) { // bswap
1601 int encode = prefix_and_encode(reg->encoding());
1602 emit_int16(0x0F, (0xC8 | encode));
1603 }
1604
1605 void Assembler::blsil(Register dst, Register src) {
1606 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1607 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1608 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1609 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1610 }
1611
1612 void Assembler::blsil(Register dst, Address src) {
1613 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1614 InstructionMark im(this);
1615 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1616 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1617 emit_int8((unsigned char)0xF3);
1618 emit_operand(rbx, src, 0);
1619 }
1620
1621 void Assembler::blsmskl(Register dst, Register src) {
1622 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1623 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1624 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1625 emit_int16((unsigned char)0xF3,
1626 0xC0 | encode);
1627 }
1628
1629 void Assembler::blsmskl(Register dst, Address src) {
1630 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1631 InstructionMark im(this);
1632 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1633 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1634 emit_int8((unsigned char)0xF3);
1635 emit_operand(rdx, src, 0);
1636 }
1637
1638 void Assembler::blsrl(Register dst, Register src) {
1639 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1640 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1641 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1642 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1643 }
1644
1645 void Assembler::blsrl(Register dst, Address src) {
1646 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1647 InstructionMark im(this);
1648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1649 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1650 emit_int8((unsigned char)0xF3);
1651 emit_operand(rcx, src, 0);
1652 }
1653
1654 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1655 // suspect disp32 is always good
1656 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1657
1658 if (L.is_bound()) {
1659 const int long_size = 5;
1660 int offs = (int)( target(L) - pc() );
1661 assert(offs <= 0, "assembler error");
1662 InstructionMark im(this);
1663 // 1110 1000 #32-bit disp
1664 emit_int8((unsigned char)0xE8);
1665 emit_data(offs - long_size, rtype, operand);
1666 } else {
1667 InstructionMark im(this);
1668 // 1110 1000 #32-bit disp
1669 L.add_patch_at(code(), locator());
1670
1671 emit_int8((unsigned char)0xE8);
1672 emit_data(int(0), rtype, operand);
1673 }
1674 }
1675
1676 void Assembler::call(Register dst) {
1677 int encode = prefix_and_encode(dst->encoding());
1678 emit_int16((unsigned char)0xFF, (0xD0 | encode));
1679 }
1680
1681
1682 void Assembler::call(Address adr) {
1683 InstructionMark im(this);
1684 prefix(adr);
1685 emit_int8((unsigned char)0xFF);
1686 emit_operand(rdx, adr, 0);
1687 }
1688
1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1690 InstructionMark im(this);
1691 emit_int8((unsigned char)0xE8);
1692 intptr_t disp = entry - (pc() + sizeof(int32_t));
1693 // Entry is NULL in case of a scratch emit.
1694 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1695 // Technically, should use call32_operand, but this format is
1696 // implied by the fact that we're emitting a call instruction.
1697
1698 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1699 emit_data((int) disp, rspec, operand);
1700 }
1701
1702 void Assembler::cdql() {
1703 emit_int8((unsigned char)0x99);
1704 }
1705
1706 void Assembler::cld() {
1707 emit_int8((unsigned char)0xFC);
1708 }
1709
1710 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1711 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1712 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1713 emit_int24(0x0F,
1714 0x40 | cc,
1715 0xC0 | encode);
1716 }
1717
1718
1719 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1720 InstructionMark im(this);
1721 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1722 prefix(src, dst);
1723 emit_int16(0x0F, (0x40 | cc));
1724 emit_operand(dst, src, 0);
1725 }
1726
1727 void Assembler::cmpb(Address dst, int imm8) {
1728 InstructionMark im(this);
1729 prefix(dst);
1730 emit_int8((unsigned char)0x80);
1731 emit_operand(rdi, dst, 1);
1732 emit_int8(imm8);
1733 }
1734
1735 void Assembler::cmpl(Address dst, int32_t imm32) {
1736 InstructionMark im(this);
1737 prefix(dst);
1738 emit_arith_operand(0x81, as_Register(7), dst, imm32);
1739 }
1740
1741 void Assembler::cmpl(Register dst, int32_t imm32) {
1742 prefix(dst);
1743 emit_arith(0x81, 0xF8, dst, imm32);
1744 }
1745
1746 void Assembler::cmpl(Register dst, Register src) {
1747 (void) prefix_and_encode(dst->encoding(), src->encoding());
1748 emit_arith(0x3B, 0xC0, dst, src);
1749 }
1750
1751 void Assembler::cmpl(Register dst, Address src) {
1752 InstructionMark im(this);
1753 prefix(src, dst);
1754 emit_int8(0x3B);
1755 emit_operand(dst, src, 0);
1756 }
1757
1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
1759 InstructionMark im(this);
1760 prefix(dst);
1761 emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
1762 }
1763
1764 void Assembler::cmpw(Address dst, int imm16) {
1765 InstructionMark im(this);
1766 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1767 emit_int16(0x66, (unsigned char)0x81);
1768 emit_operand(rdi, dst, 2);
1769 emit_int16(imm16);
1770 }
1771
1772 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1773 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1774 // The ZF is set if the compared values were equal, and cleared otherwise.
1775 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1776 InstructionMark im(this);
1777 prefix(adr, reg);
1778 emit_int16(0x0F, (unsigned char)0xB1);
1779 emit_operand(reg, adr, 0);
1780 }
1781
1782 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
1783 InstructionMark im(this);
1784 size_prefix();
1785 prefix(adr, reg);
1786 emit_int16(0x0F, (unsigned char)0xB1);
1787 emit_operand(reg, adr, 0);
1788 }
1789
1790 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1791 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1792 // The ZF is set if the compared values were equal, and cleared otherwise.
1793 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1794 InstructionMark im(this);
1795 prefix(adr, reg, true);
1796 emit_int16(0x0F, (unsigned char)0xB0);
1797 emit_operand(reg, adr, 0);
1798 }
1799
1800 void Assembler::comisd(XMMRegister dst, Address src) {
1801 // NOTE: dbx seems to decode this as comiss even though the
1802 // 0x66 is there. Strangely ucomisd comes out correct
1803 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1804 InstructionMark im(this);
1805 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1806 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1807 attributes.set_rex_vex_w_reverted();
1808 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1809 emit_int8(0x2F);
1810 emit_operand(dst, src, 0);
1811 }
1812
1813 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1814 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1815 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1816 attributes.set_rex_vex_w_reverted();
1817 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1818 emit_int16(0x2F, (0xC0 | encode));
1819 }
1820
1821 void Assembler::comiss(XMMRegister dst, Address src) {
1822 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1823 InstructionMark im(this);
1824 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1825 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1826 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1827 emit_int8(0x2F);
1828 emit_operand(dst, src, 0);
1829 }
1830
1831 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1832 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1833 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1834 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1835 emit_int16(0x2F, (0xC0 | encode));
1836 }
1837
1838 void Assembler::cpuid() {
1839 emit_int16(0x0F, (unsigned char)0xA2);
1840 }
1841
1842 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1843 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1844 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1845 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1846 //
1847 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1848 //
1849 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1850 //
1851 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1852 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1853 assert(VM_Version::supports_sse4_2(), "");
1854 int8_t w = 0x01;
1855 Prefix p = Prefix_EMPTY;
1856
1857 emit_int8((unsigned char)0xF2);
1858 switch (sizeInBytes) {
1859 case 1:
1860 w = 0;
1861 break;
1862 case 2:
1863 case 4:
1864 break;
1865 LP64_ONLY(case 8:)
1866 // This instruction is not valid in 32 bits
1867 // Note:
1868 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1869 //
1870 // Page B - 72 Vol. 2C says
1871 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1872 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1873 // F0!!!
1874 // while 3 - 208 Vol. 2A
1875 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1876 //
1877 // the 0 on a last bit is reserved for a different flavor of this instruction :
1878 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1879 p = REX_W;
1880 break;
1881 default:
1882 assert(0, "Unsupported value for a sizeInBytes argument");
1883 break;
1884 }
1885 LP64_ONLY(prefix(crc, v, p);)
1886 emit_int32(0x0F,
1887 0x38,
1888 0xF0 | w,
1889 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1890 }
1891
1892 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1893 assert(VM_Version::supports_sse4_2(), "");
1894 InstructionMark im(this);
1895 int8_t w = 0x01;
1896 Prefix p = Prefix_EMPTY;
1897
1898 emit_int8((int8_t)0xF2);
1899 switch (sizeInBytes) {
1900 case 1:
1901 w = 0;
1902 break;
1903 case 2:
1904 case 4:
1905 break;
1906 LP64_ONLY(case 8:)
1907 // This instruction is not valid in 32 bits
1908 p = REX_W;
1909 break;
1910 default:
1911 assert(0, "Unsupported value for a sizeInBytes argument");
1912 break;
1913 }
1914 LP64_ONLY(prefix(crc, adr, p);)
1915 emit_int24(0x0F, 0x38, (0xF0 | w));
1916 emit_operand(crc, adr, 0);
1917 }
1918
1919 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1920 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1921 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1922 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1923 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1924 }
1925
1926 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
1927 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1928 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1929 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1930 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1931 }
1932
1933 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
1934 assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
1935 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
1936 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
1937 emit_int24(0x1D, (0xC0 | encode), imm8);
1938 }
1939
1940 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
1941 assert(VM_Version::supports_avx512vl(), "");
1942 InstructionMark im(this);
1943 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
1944 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
1945 attributes.reset_is_clear_context();
1946 attributes.set_embedded_opmask_register_specifier(mask);
1947 attributes.set_is_evex_instruction();
1948 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
1949 emit_int8(0x1D);
1950 emit_operand(src, dst, 1);
1951 emit_int8(imm8);
1952 }
1953
1954 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
1955 assert(VM_Version::supports_avx512vl() || VM_Version::supports_f16c(), "");
1956 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
1957 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1958 emit_int16(0x13, (0xC0 | encode));
1959 }
1960
1961 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1962 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1964 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1965 emit_int16(0x5B, (0xC0 | encode));
1966 }
1967
1968 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
1969 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1970 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1971 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1972 emit_int16(0x5B, (0xC0 | encode));
1973 }
1974
1975 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1976 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1977 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1978 attributes.set_rex_vex_w_reverted();
1979 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1980 emit_int16(0x5A, (0xC0 | encode));
1981 }
1982
1983 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1984 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1985 InstructionMark im(this);
1986 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1987 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1988 attributes.set_rex_vex_w_reverted();
1989 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1990 emit_int8(0x5A);
1991 emit_operand(dst, src, 0);
1992 }
1993
1994 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1995 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1996 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1997 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1998 emit_int16(0x2A, (0xC0 | encode));
1999 }
2000
2001 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
2002 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2003 InstructionMark im(this);
2004 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2005 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2006 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2007 emit_int8(0x2A);
2008 emit_operand(dst, src, 0);
2009 }
2010
2011 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
2012 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2013 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2014 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2015 emit_int16(0x2A, (0xC0 | encode));
2016 }
2017
2018 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
2019 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2020 InstructionMark im(this);
2021 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2022 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2023 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2024 emit_int8(0x2A);
2025 emit_operand(dst, src, 0);
2026 }
2027
2028 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
2029 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2030 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2031 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2032 emit_int16(0x2A, (0xC0 | encode));
2033 }
2034
2035 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
2036 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2037 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2038 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2039 emit_int16(0x5A, (0xC0 | encode));
2040 }
2041
2042 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
2043 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2044 InstructionMark im(this);
2045 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2046 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2047 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2048 emit_int8(0x5A);
2049 emit_operand(dst, src, 0);
2050 }
2051
2052
2053 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
2054 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2055 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2056 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2057 emit_int16(0x2C, (0xC0 | encode));
2058 }
2059
2060 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
2061 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2062 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2063 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2064 emit_int16(0x2D, (0xC0 | encode));
2065 }
2066
2067 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
2068 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2069 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2070 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2071 emit_int16(0x2C, (0xC0 | encode));
2072 }
2073
2074 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
2075 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2076 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2077 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2078 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2079 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2080 }
2081
2082 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
2083 assert(VM_Version::supports_ssse3(), "");
2084 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2085 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2086 emit_int16(0x1C, (0xC0 | encode));
2087 }
2088
2089 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
2090 assert(VM_Version::supports_ssse3(), "");
2091 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2092 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2093 emit_int16(0x1D, (0xC0 | encode));
2094 }
2095
2096 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
2097 assert(VM_Version::supports_ssse3(), "");
2098 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2099 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2100 emit_int16(0x1E, (0xC0 | encode));
2101 }
2102
2103 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
2104 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2105 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2106 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
2107 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2108 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2109 emit_int16(0x1C, (0xC0 | encode));
2110 }
2111
2112 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
2113 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2114 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2115 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
2116 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2117 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2118 emit_int16(0x1D, (0xC0 | encode));
2119 }
2120
2121 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
2122 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
2123 vector_len == AVX_256bit? VM_Version::supports_avx2() :
2124 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
2125 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2126 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2127 emit_int16(0x1E, (0xC0 | encode));
2128 }
2129
2130 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
2131 assert(UseAVX > 2, "");
2132 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2133 attributes.set_is_evex_instruction();
2134 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2135 emit_int16(0x1F, (0xC0 | encode));
2136 }
2137
2138 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2139 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2140 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2141 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2142 emit_int16(0x5A, (0xC0 | encode));
2143 }
2144
2145 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2146 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2147 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2148 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2149 attributes.set_rex_vex_w_reverted();
2150 emit_int16(0x5A, (0xC0 | encode));
2151 }
2152
2153 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2154 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2155 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2156 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2157 emit_int16(0x5B, (0xC0 | encode));
2158 }
2159
2160 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2161 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2162 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2164 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2165 }
2166
2167 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2168 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2169 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2170 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2171 emit_int16(0x5B, (0xC0 | encode));
2172 }
2173
2174 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2175 assert(VM_Version::supports_avx512dq(), "");
2176 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2177 attributes.set_is_evex_instruction();
2178 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2179 emit_int16(0x7A, (0xC0 | encode));
2180 }
2181
2182 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2183 assert(VM_Version::supports_avx512dq(), "");
2184 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2185 attributes.set_is_evex_instruction();
2186 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2187 emit_int16(0x7B, (0xC0 | encode));
2188 }
2189
2190 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2191 assert(VM_Version::supports_avx512dq(), "");
2192 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2193 attributes.set_is_evex_instruction();
2194 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2195 emit_int16(0x5B, (0xC0 | encode));
2196 }
2197
2198 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2199 assert(VM_Version::supports_avx512dq(), "");
2200 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2201 attributes.set_is_evex_instruction();
2202 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2203 emit_int16(0x7A, (0xC0 | encode));
2204 }
2205
2206 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2207 assert(VM_Version::supports_avx512dq(), "");
2208 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2209 attributes.set_is_evex_instruction();
2210 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2211 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2212 }
2213
2214 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
2215 assert(VM_Version::supports_avx512bw(), "");
2216 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2217 attributes.set_is_evex_instruction();
2218 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2219 emit_int16(0x30, (0xC0 | encode));
2220 }
2221
2222 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
2223 assert(UseAVX > 2, "");
2224 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2225 attributes.set_is_evex_instruction();
2226 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2227 emit_int16(0x33, (0xC0 | encode));
2228 }
2229
2230 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
2231 assert(UseAVX > 2, "");
2232 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2233 attributes.set_is_evex_instruction();
2234 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2235 emit_int16(0x31, (0xC0 | encode));
2236 }
2237
2238 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
2239 assert(UseAVX > 2, "");
2240 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2241 attributes.set_is_evex_instruction();
2242 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2243 emit_int16(0x35, (0xC0 | encode));
2244 }
2245
2246 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
2247 assert(UseAVX > 2, "");
2248 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2249 attributes.set_is_evex_instruction();
2250 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2251 emit_int16(0x32, (0xC0 | encode));
2252 }
2253
2254 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
2255 assert(UseAVX > 2, "");
2256 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2257 attributes.set_is_evex_instruction();
2258 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2259 emit_int16(0x34, (0xC0 | encode));
2260 }
2261
2262 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
2263 assert(UseAVX > 2, "");
2264 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2265 attributes.set_is_evex_instruction();
2266 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2267 emit_int16(0x25, (0xC0 | encode));
2268 }
2269
2270 void Assembler::decl(Address dst) {
2271 // Don't use it directly. Use MacroAssembler::decrement() instead.
2272 InstructionMark im(this);
2273 prefix(dst);
2274 emit_int8((unsigned char)0xFF);
2275 emit_operand(rcx, dst, 0);
2276 }
2277
2278 void Assembler::divsd(XMMRegister dst, Address src) {
2279 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2280 InstructionMark im(this);
2281 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2282 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2283 attributes.set_rex_vex_w_reverted();
2284 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2285 emit_int8(0x5E);
2286 emit_operand(dst, src, 0);
2287 }
2288
2289 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
2290 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2291 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2292 attributes.set_rex_vex_w_reverted();
2293 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2294 emit_int16(0x5E, (0xC0 | encode));
2295 }
2296
2297 void Assembler::divss(XMMRegister dst, Address src) {
2298 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2299 InstructionMark im(this);
2300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2301 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2302 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2303 emit_int8(0x5E);
2304 emit_operand(dst, src, 0);
2305 }
2306
2307 void Assembler::divss(XMMRegister dst, XMMRegister src) {
2308 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2309 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2310 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2311 emit_int16(0x5E, (0xC0 | encode));
2312 }
2313
2314 void Assembler::hlt() {
2315 emit_int8((unsigned char)0xF4);
2316 }
2317
2318 void Assembler::idivl(Register src) {
2319 int encode = prefix_and_encode(src->encoding());
2320 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2321 }
2322
2323 void Assembler::divl(Register src) { // Unsigned
2324 int encode = prefix_and_encode(src->encoding());
2325 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2326 }
2327
2328 void Assembler::imull(Register src) {
2329 int encode = prefix_and_encode(src->encoding());
2330 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2331 }
2332
2333 void Assembler::imull(Register dst, Register src) {
2334 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2335 emit_int24(0x0F,
2336 (unsigned char)0xAF,
2337 (0xC0 | encode));
2338 }
2339
2340 void Assembler::imull(Register dst, Address src, int32_t value) {
2341 InstructionMark im(this);
2342 prefix(src, dst);
2343 if (is8bit(value)) {
2344 emit_int8((unsigned char)0x6B);
2345 emit_operand(dst, src, 1);
2346 emit_int8(value);
2347 } else {
2348 emit_int8((unsigned char)0x69);
2349 emit_operand(dst, src, 4);
2350 emit_int32(value);
2351 }
2352 }
2353
2354 void Assembler::imull(Register dst, Register src, int value) {
2355 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2356 if (is8bit(value)) {
2357 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2358 } else {
2359 emit_int16(0x69, (0xC0 | encode));
2360 emit_int32(value);
2361 }
2362 }
2363
2364 void Assembler::imull(Register dst, Address src) {
2365 InstructionMark im(this);
2366 prefix(src, dst);
2367 emit_int16(0x0F, (unsigned char)0xAF);
2368 emit_operand(dst, src, 0);
2369 }
2370
2371
2372 void Assembler::incl(Address dst) {
2373 // Don't use it directly. Use MacroAssembler::increment() instead.
2374 InstructionMark im(this);
2375 prefix(dst);
2376 emit_int8((unsigned char)0xFF);
2377 emit_operand(rax, dst, 0);
2378 }
2379
2380 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2381 InstructionMark im(this);
2382 assert((0 <= cc) && (cc < 16), "illegal cc");
2383 if (L.is_bound()) {
2384 address dst = target(L);
2385 assert(dst != NULL, "jcc most probably wrong");
2386
2387 const int short_size = 2;
2388 const int long_size = 6;
2389 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2390 if (maybe_short && is8bit(offs - short_size)) {
2391 // 0111 tttn #8-bit disp
2392 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2393 } else {
2394 // 0000 1111 1000 tttn #32-bit disp
2395 assert(is_simm32(offs - long_size),
2396 "must be 32bit offset (call4)");
2397 emit_int16(0x0F, (0x80 | cc));
2398 emit_int32(offs - long_size);
2399 }
2400 } else {
2401 // Note: could eliminate cond. jumps to this jump if condition
2402 // is the same however, seems to be rather unlikely case.
2403 // Note: use jccb() if label to be bound is very close to get
2404 // an 8-bit displacement
2405 L.add_patch_at(code(), locator());
2406 emit_int16(0x0F, (0x80 | cc));
2407 emit_int32(0);
2408 }
2409 }
2410
2411 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2412 if (L.is_bound()) {
2413 const int short_size = 2;
2414 address entry = target(L);
2415 #ifdef ASSERT
2416 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2417 intptr_t delta = short_branch_delta();
2418 if (delta != 0) {
2419 dist += (dist < 0 ? (-delta) :delta);
2420 }
2421 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2422 #endif
2423 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2424 // 0111 tttn #8-bit disp
2425 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2426 } else {
2427 InstructionMark im(this);
2428 L.add_patch_at(code(), locator(), file, line);
2429 emit_int16(0x70 | cc, 0);
2430 }
2431 }
2432
2433 void Assembler::jmp(Address adr) {
2434 InstructionMark im(this);
2435 prefix(adr);
2436 emit_int8((unsigned char)0xFF);
2437 emit_operand(rsp, adr, 0);
2438 }
2439
2440 void Assembler::jmp(Label& L, bool maybe_short) {
2441 if (L.is_bound()) {
2442 address entry = target(L);
2443 assert(entry != NULL, "jmp most probably wrong");
2444 InstructionMark im(this);
2445 const int short_size = 2;
2446 const int long_size = 5;
2447 intptr_t offs = entry - pc();
2448 if (maybe_short && is8bit(offs - short_size)) {
2449 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
2450 } else {
2451 emit_int8((unsigned char)0xE9);
2452 emit_int32(offs - long_size);
2453 }
2454 } else {
2455 // By default, forward jumps are always 32-bit displacements, since
2456 // we can't yet know where the label will be bound. If you're sure that
2457 // the forward jump will not run beyond 256 bytes, use jmpb to
2458 // force an 8-bit displacement.
2459 InstructionMark im(this);
2460 L.add_patch_at(code(), locator());
2461 emit_int8((unsigned char)0xE9);
2462 emit_int32(0);
2463 }
2464 }
2465
2466 void Assembler::jmp(Register entry) {
2467 int encode = prefix_and_encode(entry->encoding());
2468 emit_int16((unsigned char)0xFF, (0xE0 | encode));
2469 }
2470
2471 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2472 InstructionMark im(this);
2473 emit_int8((unsigned char)0xE9);
2474 assert(dest != NULL, "must have a target");
2475 intptr_t disp = dest - (pc() + sizeof(int32_t));
2476 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2477 emit_data(disp, rspec.reloc(), call32_operand);
2478 }
2479
2480 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2481 if (L.is_bound()) {
2482 const int short_size = 2;
2483 address entry = target(L);
2484 assert(entry != NULL, "jmp most probably wrong");
2485 #ifdef ASSERT
2486 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2487 intptr_t delta = short_branch_delta();
2488 if (delta != 0) {
2489 dist += (dist < 0 ? (-delta) :delta);
2490 }
2491 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2492 #endif
2493 intptr_t offs = entry - pc();
2494 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
2495 } else {
2496 InstructionMark im(this);
2497 L.add_patch_at(code(), locator(), file, line);
2498 emit_int16((unsigned char)0xEB, 0);
2499 }
2500 }
2501
2502 void Assembler::ldmxcsr( Address src) {
2503 if (UseAVX > 0 ) {
2504 InstructionMark im(this);
2505 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2506 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2507 emit_int8((unsigned char)0xAE);
2508 emit_operand(as_Register(2), src, 0);
2509 } else {
2510 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2511 InstructionMark im(this);
2512 prefix(src);
2513 emit_int16(0x0F, (unsigned char)0xAE);
2514 emit_operand(as_Register(2), src, 0);
2515 }
2516 }
2517
2518 void Assembler::leal(Register dst, Address src) {
2519 InstructionMark im(this);
2520 prefix(src, dst);
2521 emit_int8((unsigned char)0x8D);
2522 emit_operand(dst, src, 0);
2523 }
2524
2525 void Assembler::lfence() {
2526 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
2527 }
2528
2529 void Assembler::lock() {
2530 emit_int8((unsigned char)0xF0);
2531 }
2532
2533 void Assembler::size_prefix() {
2534 emit_int8(0x66);
2535 }
2536
2537 void Assembler::lzcntl(Register dst, Register src) {
2538 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2539 emit_int8((unsigned char)0xF3);
2540 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2541 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
2542 }
2543
2544 void Assembler::lzcntl(Register dst, Address src) {
2545 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2546 InstructionMark im(this);
2547 emit_int8((unsigned char)0xF3);
2548 prefix(src, dst);
2549 emit_int16(0x0F, (unsigned char)0xBD);
2550 emit_operand(dst, src, 0);
2551 }
2552
2553 // Emit mfence instruction
2554 void Assembler::mfence() {
2555 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2556 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
2557 }
2558
2559 // Emit sfence instruction
2560 void Assembler::sfence() {
2561 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2562 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
2563 }
2564
2565 void Assembler::mov(Register dst, Register src) {
2566 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2567 }
2568
2569 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2570 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2571 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2572 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2573 attributes.set_rex_vex_w_reverted();
2574 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2575 emit_int16(0x28, (0xC0 | encode));
2576 }
2577
2578 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2579 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2580 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2581 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2582 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2583 emit_int16(0x28, (0xC0 | encode));
2584 }
2585
2586 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2587 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2588 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2589 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2590 emit_int16(0x16, (0xC0 | encode));
2591 }
2592
2593 void Assembler::movb(Register dst, Address src) {
2594 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2595 InstructionMark im(this);
2596 prefix(src, dst, true);
2597 emit_int8((unsigned char)0x8A);
2598 emit_operand(dst, src, 0);
2599 }
2600
2601 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2602 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2603 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2604 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2605 attributes.set_rex_vex_w_reverted();
2606 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2607 emit_int16(0x12, 0xC0 | encode);
2608 }
2609
2610 void Assembler::movddup(XMMRegister dst, Address src) {
2611 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2612 InstructionMark im(this);
2613 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2614 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
2615 attributes.set_rex_vex_w_reverted();
2616 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2617 emit_int8(0x12);
2618 emit_operand(dst, src, 0);
2619 }
2620
2621 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
2622 assert(VM_Version::supports_avx(), "");
2623 InstructionMark im(this);
2624 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2625 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
2626 attributes.set_rex_vex_w_reverted();
2627 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2628 emit_int8(0x12);
2629 emit_operand(dst, src, 0);
2630 }
2631
2632 void Assembler::kmovbl(KRegister dst, KRegister src) {
2633 assert(VM_Version::supports_avx512dq(), "");
2634 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2635 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2636 emit_int16((unsigned char)0x90, (0xC0 | encode));
2637 }
2638
2639 void Assembler::kmovbl(KRegister dst, Register src) {
2640 assert(VM_Version::supports_avx512dq(), "");
2641 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2642 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2643 emit_int16((unsigned char)0x92, (0xC0 | encode));
2644 }
2645
2646 void Assembler::kmovbl(Register dst, KRegister src) {
2647 assert(VM_Version::supports_avx512dq(), "");
2648 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2649 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2650 emit_int16((unsigned char)0x93, (0xC0 | encode));
2651 }
2652
2653 void Assembler::kmovwl(KRegister dst, Register src) {
2654 assert(VM_Version::supports_evex(), "");
2655 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2656 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2657 emit_int16((unsigned char)0x92, (0xC0 | encode));
2658 }
2659
2660 void Assembler::kmovwl(Register dst, KRegister src) {
2661 assert(VM_Version::supports_evex(), "");
2662 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2663 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2664 emit_int16((unsigned char)0x93, (0xC0 | encode));
2665 }
2666
2667 void Assembler::kmovwl(KRegister dst, Address src) {
2668 assert(VM_Version::supports_evex(), "");
2669 InstructionMark im(this);
2670 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2671 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2672 emit_int8((unsigned char)0x90);
2673 emit_operand(dst, src, 0);
2674 }
2675
2676 void Assembler::kmovwl(Address dst, KRegister src) {
2677 assert(VM_Version::supports_evex(), "");
2678 InstructionMark im(this);
2679 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2680 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2681 emit_int8((unsigned char)0x91);
2682 emit_operand(src, dst, 0);
2683 }
2684
2685 void Assembler::kmovwl(KRegister dst, KRegister src) {
2686 assert(VM_Version::supports_evex(), "");
2687 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2688 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2689 emit_int16((unsigned char)0x90, (0xC0 | encode));
2690 }
2691
2692 void Assembler::kmovdl(KRegister dst, Register src) {
2693 assert(VM_Version::supports_avx512bw(), "");
2694 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2695 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2696 emit_int16((unsigned char)0x92, (0xC0 | encode));
2697 }
2698
2699 void Assembler::kmovdl(Register dst, KRegister src) {
2700 assert(VM_Version::supports_avx512bw(), "");
2701 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2702 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2703 emit_int16((unsigned char)0x93, (0xC0 | encode));
2704 }
2705
2706 void Assembler::kmovql(KRegister dst, KRegister src) {
2707 assert(VM_Version::supports_avx512bw(), "");
2708 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2709 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2710 emit_int16((unsigned char)0x90, (0xC0 | encode));
2711 }
2712
2713 void Assembler::kmovql(KRegister dst, Address src) {
2714 assert(VM_Version::supports_avx512bw(), "");
2715 InstructionMark im(this);
2716 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2717 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2718 emit_int8((unsigned char)0x90);
2719 emit_operand(dst, src, 0);
2720 }
2721
2722 void Assembler::kmovql(Address dst, KRegister src) {
2723 assert(VM_Version::supports_avx512bw(), "");
2724 InstructionMark im(this);
2725 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2726 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2727 emit_int8((unsigned char)0x91);
2728 emit_operand(src, dst, 0);
2729 }
2730
2731 void Assembler::kmovql(KRegister dst, Register src) {
2732 assert(VM_Version::supports_avx512bw(), "");
2733 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2734 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2735 emit_int16((unsigned char)0x92, (0xC0 | encode));
2736 }
2737
2738 void Assembler::kmovql(Register dst, KRegister src) {
2739 assert(VM_Version::supports_avx512bw(), "");
2740 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2741 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2742 emit_int16((unsigned char)0x93, (0xC0 | encode));
2743 }
2744
2745 void Assembler::knotwl(KRegister dst, KRegister src) {
2746 assert(VM_Version::supports_evex(), "");
2747 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2748 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2749 emit_int16(0x44, (0xC0 | encode));
2750 }
2751
2752 void Assembler::knotbl(KRegister dst, KRegister src) {
2753 assert(VM_Version::supports_evex(), "");
2754 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2755 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2756 emit_int16(0x44, (0xC0 | encode));
2757 }
2758
2759 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
2760 assert(VM_Version::supports_avx512dq(), "");
2761 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2762 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2763 emit_int16(0x45, (0xC0 | encode));
2764 }
2765
2766 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
2767 assert(VM_Version::supports_evex(), "");
2768 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2769 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2770 emit_int16(0x45, (0xC0 | encode));
2771 }
2772
2773 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
2774 assert(VM_Version::supports_avx512bw(), "");
2775 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2776 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2777 emit_int16(0x45, (0xC0 | encode));
2778 }
2779
2780 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
2781 assert(VM_Version::supports_avx512bw(), "");
2782 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2783 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2784 emit_int16(0x45, (0xC0 | encode));
2785 }
2786
2787 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
2788 assert(VM_Version::supports_avx512dq(), "");
2789 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2790 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2791 emit_int16(0x47, (0xC0 | encode));
2792 }
2793
2794 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
2795 assert(VM_Version::supports_evex(), "");
2796 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2797 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2798 emit_int16(0x47, (0xC0 | encode));
2799 }
2800
2801 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
2802 assert(VM_Version::supports_avx512bw(), "");
2803 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2804 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2805 emit_int16(0x47, (0xC0 | encode));
2806 }
2807
2808 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
2809 assert(VM_Version::supports_avx512bw(), "");
2810 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2811 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2812 emit_int16(0x47, (0xC0 | encode));
2813 }
2814
2815 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
2816 assert(VM_Version::supports_avx512dq(), "");
2817 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2818 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2819 emit_int16(0x41, (0xC0 | encode));
2820 }
2821
2822 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
2823 assert(VM_Version::supports_evex(), "");
2824 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2825 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2826 emit_int16(0x41, (0xC0 | encode));
2827 }
2828
2829 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
2830 assert(VM_Version::supports_avx512bw(), "");
2831 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2832 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2833 emit_int16(0x41, (0xC0 | encode));
2834 }
2835
2836 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
2837 assert(VM_Version::supports_avx512bw(), "");
2838 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2839 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2840 emit_int16(0x41, (0xC0 | encode));
2841 }
2842
2843 void Assembler::knotdl(KRegister dst, KRegister src) {
2844 assert(VM_Version::supports_avx512bw(), "");
2845 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2846 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2847 emit_int16(0x44, (0xC0 | encode));
2848 }
2849
2850 void Assembler::knotql(KRegister dst, KRegister src) {
2851 assert(VM_Version::supports_avx512bw(), "");
2852 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2853 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2854 emit_int16(0x44, (0xC0 | encode));
2855 }
2856
2857 // This instruction produces ZF or CF flags
2858 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2859 assert(VM_Version::supports_avx512dq(), "");
2860 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2861 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2862 emit_int16((unsigned char)0x98, (0xC0 | encode));
2863 }
2864
2865 // This instruction produces ZF or CF flags
2866 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2867 assert(VM_Version::supports_evex(), "");
2868 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2869 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2870 emit_int16((unsigned char)0x98, (0xC0 | encode));
2871 }
2872
2873 // This instruction produces ZF or CF flags
2874 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2875 assert(VM_Version::supports_avx512bw(), "");
2876 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2877 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2878 emit_int16((unsigned char)0x98, (0xC0 | encode));
2879 }
2880
2881 // This instruction produces ZF or CF flags
2882 void Assembler::kortestql(KRegister src1, KRegister src2) {
2883 assert(VM_Version::supports_avx512bw(), "");
2884 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2885 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2886 emit_int16((unsigned char)0x98, (0xC0 | encode));
2887 }
2888
2889 // This instruction produces ZF or CF flags
2890 void Assembler::ktestql(KRegister src1, KRegister src2) {
2891 assert(VM_Version::supports_avx512bw(), "");
2892 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2893 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2894 emit_int16((unsigned char)0x99, (0xC0 | encode));
2895 }
2896
2897 void Assembler::ktestdl(KRegister src1, KRegister src2) {
2898 assert(VM_Version::supports_avx512bw(), "");
2899 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2900 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2901 emit_int16((unsigned char)0x99, (0xC0 | encode));
2902 }
2903
2904 void Assembler::ktestwl(KRegister src1, KRegister src2) {
2905 assert(VM_Version::supports_avx512dq(), "");
2906 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2907 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2908 emit_int16((unsigned char)0x99, (0xC0 | encode));
2909 }
2910
2911 void Assembler::ktestbl(KRegister src1, KRegister src2) {
2912 assert(VM_Version::supports_avx512dq(), "");
2913 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2914 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2915 emit_int16((unsigned char)0x99, (0xC0 | encode));
2916 }
2917
2918 void Assembler::ktestq(KRegister src1, KRegister src2) {
2919 assert(VM_Version::supports_avx512bw(), "");
2920 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2921 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2922 emit_int16((unsigned char)0x99, (0xC0 | encode));
2923 }
2924
2925 void Assembler::ktestd(KRegister src1, KRegister src2) {
2926 assert(VM_Version::supports_avx512bw(), "");
2927 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2928 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2929 emit_int16((unsigned char)0x99, (0xC0 | encode));
2930 }
2931
2932 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
2933 assert(VM_Version::supports_avx512dq(), "");
2934 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2935 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2936 emit_int16(0x46, (0xC0 | encode));
2937 }
2938
2939 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
2940 assert(VM_Version::supports_avx512dq(), "");
2941 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2942 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2943 emit_int16(0x32, (0xC0 | encode));
2944 emit_int8(imm8);
2945 }
2946
2947 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
2948 assert(VM_Version::supports_avx512bw(), "");
2949 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2950 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2951 emit_int16(0x33, (0xC0 | encode));
2952 emit_int8(imm8);
2953 }
2954
2955
2956 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
2957 assert(VM_Version::supports_avx512dq(), "");
2958 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2959 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2960 emit_int16(0x30, (0xC0 | encode));
2961 }
2962
2963 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
2964 assert(VM_Version::supports_evex(), "");
2965 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2966 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2967 emit_int16(0x30, (0xC0 | encode));
2968 emit_int8(imm8);
2969 }
2970
2971 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
2972 assert(VM_Version::supports_avx512bw(), "");
2973 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2974 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2975 emit_int16(0x31, (0xC0 | encode));
2976 emit_int8(imm8);
2977 }
2978
2979 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
2980 assert(VM_Version::supports_avx512bw(), "");
2981 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2982 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2983 emit_int16(0x31, (0xC0 | encode));
2984 emit_int8(imm8);
2985 }
2986
2987 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
2988 assert(VM_Version::supports_avx512bw(), "");
2989 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2990 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2991 emit_int16(0x4B, (0xC0 | encode));
2992 }
2993
2994 void Assembler::movb(Address dst, int imm8) {
2995 InstructionMark im(this);
2996 prefix(dst);
2997 emit_int8((unsigned char)0xC6);
2998 emit_operand(rax, dst, 1);
2999 emit_int8(imm8);
3000 }
3001
3002
3003 void Assembler::movb(Address dst, Register src) {
3004 assert(src->has_byte_register(), "must have byte register");
3005 InstructionMark im(this);
3006 prefix(dst, src, true);
3007 emit_int8((unsigned char)0x88);
3008 emit_operand(src, dst, 0);
3009 }
3010
3011 void Assembler::movdl(XMMRegister dst, Register src) {
3012 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3013 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3014 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3015 emit_int16(0x6E, (0xC0 | encode));
3016 }
3017
3018 void Assembler::movdl(Register dst, XMMRegister src) {
3019 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3020 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3021 // swap src/dst to get correct prefix
3022 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3023 emit_int16(0x7E, (0xC0 | encode));
3024 }
3025
3026 void Assembler::movdl(XMMRegister dst, Address src) {
3027 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3028 InstructionMark im(this);
3029 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3030 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3031 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3032 emit_int8(0x6E);
3033 emit_operand(dst, src, 0);
3034 }
3035
3036 void Assembler::movdl(Address dst, XMMRegister src) {
3037 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3038 InstructionMark im(this);
3039 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3040 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3041 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3042 emit_int8(0x7E);
3043 emit_operand(src, dst, 0);
3044 }
3045
3046 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
3047 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3048 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3049 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3050 emit_int16(0x6F, (0xC0 | encode));
3051 }
3052
3053 void Assembler::movdqa(XMMRegister dst, Address src) {
3054 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3055 InstructionMark im(this);
3056 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3057 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3058 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3059 emit_int8(0x6F);
3060 emit_operand(dst, src, 0);
3061 }
3062
3063 void Assembler::movdqu(XMMRegister dst, Address src) {
3064 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3065 InstructionMark im(this);
3066 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3067 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3068 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3069 emit_int8(0x6F);
3070 emit_operand(dst, src, 0);
3071 }
3072
3073 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
3074 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3075 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3076 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3077 emit_int16(0x6F, (0xC0 | encode));
3078 }
3079
3080 void Assembler::movdqu(Address dst, XMMRegister src) {
3081 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3082 InstructionMark im(this);
3083 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3084 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3085 attributes.reset_is_clear_context();
3086 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3087 emit_int8(0x7F);
3088 emit_operand(src, dst, 0);
3089 }
3090
3091 // Move Unaligned 256bit Vector
3092 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3093 assert(UseAVX > 0, "");
3094 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3095 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3096 emit_int16(0x6F, (0xC0 | encode));
3097 }
3098
3099 void Assembler::vmovdqu(XMMRegister dst, Address src) {
3100 assert(UseAVX > 0, "");
3101 InstructionMark im(this);
3102 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3103 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3104 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3105 emit_int8(0x6F);
3106 emit_operand(dst, src, 0);
3107 }
3108
3109 void Assembler::vmovdqu(Address dst, XMMRegister src) {
3110 assert(UseAVX > 0, "");
3111 InstructionMark im(this);
3112 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3113 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3114 attributes.reset_is_clear_context();
3115 // swap src<->dst for encoding
3116 assert(src != xnoreg, "sanity");
3117 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3118 emit_int8(0x7F);
3119 emit_operand(src, dst, 0);
3120 }
3121
3122 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3123 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3124 InstructionMark im(this);
3125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3126 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3127 emit_int8((unsigned char)0x8C);
3128 emit_operand(dst, src, 0);
3129 }
3130
3131 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3132 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3133 InstructionMark im(this);
3134 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3135 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3136 emit_int8((unsigned char)0x8C);
3137 emit_operand(dst, src, 0);
3138 }
3139
3140 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3141 assert(UseAVX > 0, "requires some form of AVX");
3142 InstructionMark im(this);
3143 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3144 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3145 emit_int8(0x2C);
3146 emit_operand(dst, src, 0);
3147 }
3148
3149 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3150 assert(UseAVX > 0, "requires some form of AVX");
3151 InstructionMark im(this);
3152 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3153 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3154 emit_int8(0x2D);
3155 emit_operand(dst, src, 0);
3156 }
3157
3158 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3159 assert(UseAVX > 0, "");
3160 InstructionMark im(this);
3161 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3162 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3163 emit_int8(0x2E);
3164 emit_operand(src, dst, 0);
3165 }
3166
3167 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3168 assert(UseAVX > 0, "");
3169 InstructionMark im(this);
3170 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3171 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3172 emit_int8(0x2F);
3173 emit_operand(src, dst, 0);
3174 }
3175
3176 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
3177 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3178 assert(VM_Version::supports_avx512vlbw(), "");
3179 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3180 attributes.set_embedded_opmask_register_specifier(mask);
3181 attributes.set_is_evex_instruction();
3182 if (merge) {
3183 attributes.reset_is_clear_context();
3184 }
3185 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3186 emit_int16(0x6F, (0xC0 | encode));
3187 }
3188
3189 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
3190 // Unmasked instruction
3191 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3192 }
3193
3194 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3195 assert(VM_Version::supports_avx512vlbw(), "");
3196 InstructionMark im(this);
3197 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3198 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3199 attributes.set_embedded_opmask_register_specifier(mask);
3200 attributes.set_is_evex_instruction();
3201 if (merge) {
3202 attributes.reset_is_clear_context();
3203 }
3204 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3205 emit_int8(0x6F);
3206 emit_operand(dst, src, 0);
3207 }
3208
3209 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
3210 // Unmasked instruction
3211 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3212 }
3213
3214 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3215 assert(VM_Version::supports_avx512vlbw(), "");
3216 assert(src != xnoreg, "sanity");
3217 InstructionMark im(this);
3218 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3219 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3220 attributes.set_embedded_opmask_register_specifier(mask);
3221 attributes.set_is_evex_instruction();
3222 if (merge) {
3223 attributes.reset_is_clear_context();
3224 }
3225 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3226 emit_int8(0x7F);
3227 emit_operand(src, dst, 0);
3228 }
3229
3230 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
3231 // Unmasked instruction
3232 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3233 }
3234
3235 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3236 assert(VM_Version::supports_avx512vlbw(), "");
3237 InstructionMark im(this);
3238 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3239 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3240 attributes.set_embedded_opmask_register_specifier(mask);
3241 attributes.set_is_evex_instruction();
3242 if (merge) {
3243 attributes.reset_is_clear_context();
3244 }
3245 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3246 emit_int8(0x6F);
3247 emit_operand(dst, src, 0);
3248 }
3249
3250 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
3251 // Unmasked instruction
3252 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3253 }
3254
3255 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3256 assert(VM_Version::supports_avx512vlbw(), "");
3257 assert(src != xnoreg, "sanity");
3258 InstructionMark im(this);
3259 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3260 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3261 attributes.set_embedded_opmask_register_specifier(mask);
3262 attributes.set_is_evex_instruction();
3263 if (merge) {
3264 attributes.reset_is_clear_context();
3265 }
3266 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3267 emit_int8(0x7F);
3268 emit_operand(src, dst, 0);
3269 }
3270
3271 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
3272 // Unmasked instruction
3273 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3274 }
3275
3276 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3277 assert(VM_Version::supports_evex(), "");
3278 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3279 attributes.set_embedded_opmask_register_specifier(mask);
3280 attributes.set_is_evex_instruction();
3281 if (merge) {
3282 attributes.reset_is_clear_context();
3283 }
3284 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3285 emit_int16(0x6F, (0xC0 | encode));
3286 }
3287
3288 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
3289 // Unmasked instruction
3290 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3291 }
3292
3293 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3294 assert(VM_Version::supports_evex(), "");
3295 InstructionMark im(this);
3296 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
3297 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3298 attributes.set_embedded_opmask_register_specifier(mask);
3299 attributes.set_is_evex_instruction();
3300 if (merge) {
3301 attributes.reset_is_clear_context();
3302 }
3303 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3304 emit_int8(0x6F);
3305 emit_operand(dst, src, 0);
3306 }
3307
3308 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
3309 // Unmasked isntruction
3310 evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
3311 }
3312
3313 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3314 assert(VM_Version::supports_evex(), "");
3315 assert(src != xnoreg, "sanity");
3316 InstructionMark im(this);
3317 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3318 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3319 attributes.set_embedded_opmask_register_specifier(mask);
3320 attributes.set_is_evex_instruction();
3321 if (merge) {
3322 attributes.reset_is_clear_context();
3323 }
3324 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3325 emit_int8(0x7F);
3326 emit_operand(src, dst, 0);
3327 }
3328
3329 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
3330 // Unmasked instruction
3331 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3332 }
3333
3334 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3335 assert(VM_Version::supports_evex(), "");
3336 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3337 attributes.set_embedded_opmask_register_specifier(mask);
3338 attributes.set_is_evex_instruction();
3339 if (merge) {
3340 attributes.reset_is_clear_context();
3341 }
3342 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3343 emit_int16(0x6F, (0xC0 | encode));
3344 }
3345
3346 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
3347 // Unmasked instruction
3348 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3349 }
3350
3351 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3352 assert(VM_Version::supports_evex(), "");
3353 InstructionMark im(this);
3354 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3355 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3356 attributes.set_embedded_opmask_register_specifier(mask);
3357 attributes.set_is_evex_instruction();
3358 if (merge) {
3359 attributes.reset_is_clear_context();
3360 }
3361 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3362 emit_int8(0x6F);
3363 emit_operand(dst, src, 0);
3364 }
3365
3366 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
3367 // Unmasked instruction
3368 evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
3369 }
3370
3371 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3372 assert(VM_Version::supports_evex(), "");
3373 assert(src != xnoreg, "sanity");
3374 InstructionMark im(this);
3375 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3376 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3377 attributes.set_embedded_opmask_register_specifier(mask);
3378 if (merge) {
3379 attributes.reset_is_clear_context();
3380 }
3381 attributes.set_is_evex_instruction();
3382 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3383 emit_int8(0x7F);
3384 emit_operand(src, dst, 0);
3385 }
3386
3387 // Uses zero extension on 64bit
3388
3389 void Assembler::movl(Register dst, int32_t imm32) {
3390 int encode = prefix_and_encode(dst->encoding());
3391 emit_int8(0xB8 | encode);
3392 emit_int32(imm32);
3393 }
3394
3395 void Assembler::movl(Register dst, Register src) {
3396 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3397 emit_int16((unsigned char)0x8B, (0xC0 | encode));
3398 }
3399
3400 void Assembler::movl(Register dst, Address src) {
3401 InstructionMark im(this);
3402 prefix(src, dst);
3403 emit_int8((unsigned char)0x8B);
3404 emit_operand(dst, src, 0);
3405 }
3406
3407 void Assembler::movl(Address dst, int32_t imm32) {
3408 InstructionMark im(this);
3409 prefix(dst);
3410 emit_int8((unsigned char)0xC7);
3411 emit_operand(rax, dst, 4);
3412 emit_int32(imm32);
3413 }
3414
3415 void Assembler::movl(Address dst, Register src) {
3416 InstructionMark im(this);
3417 prefix(dst, src);
3418 emit_int8((unsigned char)0x89);
3419 emit_operand(src, dst, 0);
3420 }
3421
3422 // New cpus require to use movsd and movss to avoid partial register stall
3423 // when loading from memory. But for old Opteron use movlpd instead of movsd.
3424 // The selection is done in MacroAssembler::movdbl() and movflt().
3425 void Assembler::movlpd(XMMRegister dst, Address src) {
3426 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3427 InstructionMark im(this);
3428 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3429 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3430 attributes.set_rex_vex_w_reverted();
3431 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3432 emit_int8(0x12);
3433 emit_operand(dst, src, 0);
3434 }
3435
3436 void Assembler::movq(XMMRegister dst, Address src) {
3437 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3438 InstructionMark im(this);
3439 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3440 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3441 attributes.set_rex_vex_w_reverted();
3442 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3443 emit_int8(0x7E);
3444 emit_operand(dst, src, 0);
3445 }
3446
3447 void Assembler::movq(Address dst, XMMRegister src) {
3448 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3449 InstructionMark im(this);
3450 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3451 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3452 attributes.set_rex_vex_w_reverted();
3453 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3454 emit_int8((unsigned char)0xD6);
3455 emit_operand(src, dst, 0);
3456 }
3457
3458 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3459 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3460 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3461 attributes.set_rex_vex_w_reverted();
3462 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3463 emit_int16((unsigned char)0xD6, (0xC0 | encode));
3464 }
3465
3466 void Assembler::movq(Register dst, XMMRegister src) {
3467 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3468 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3469 // swap src/dst to get correct prefix
3470 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3471 emit_int16(0x7E, (0xC0 | encode));
3472 }
3473
3474 void Assembler::movq(XMMRegister dst, Register src) {
3475 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3476 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3477 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3478 emit_int16(0x6E, (0xC0 | encode));
3479 }
3480
3481 void Assembler::movsbl(Register dst, Address src) { // movsxb
3482 InstructionMark im(this);
3483 prefix(src, dst);
3484 emit_int16(0x0F, (unsigned char)0xBE);
3485 emit_operand(dst, src, 0);
3486 }
3487
3488 void Assembler::movsbl(Register dst, Register src) { // movsxb
3489 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3490 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3491 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
3492 }
3493
3494 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3495 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3496 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3497 attributes.set_rex_vex_w_reverted();
3498 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3499 emit_int16(0x10, (0xC0 | encode));
3500 }
3501
3502 void Assembler::movsd(XMMRegister dst, Address src) {
3503 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3504 InstructionMark im(this);
3505 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3506 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3507 attributes.set_rex_vex_w_reverted();
3508 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3509 emit_int8(0x10);
3510 emit_operand(dst, src, 0);
3511 }
3512
3513 void Assembler::movsd(Address dst, XMMRegister src) {
3514 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3515 InstructionMark im(this);
3516 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3517 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3518 attributes.reset_is_clear_context();
3519 attributes.set_rex_vex_w_reverted();
3520 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3521 emit_int8(0x11);
3522 emit_operand(src, dst, 0);
3523 }
3524
3525 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3526 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3527 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3528 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3529 emit_int16(0x10, (0xC0 | encode));
3530 }
3531
3532 void Assembler::movss(XMMRegister dst, Address src) {
3533 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3534 InstructionMark im(this);
3535 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3536 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3537 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3538 emit_int8(0x10);
3539 emit_operand(dst, src, 0);
3540 }
3541
3542 void Assembler::movss(Address dst, XMMRegister src) {
3543 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3544 InstructionMark im(this);
3545 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3546 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3547 attributes.reset_is_clear_context();
3548 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3549 emit_int8(0x11);
3550 emit_operand(src, dst, 0);
3551 }
3552
3553 void Assembler::movswl(Register dst, Address src) { // movsxw
3554 InstructionMark im(this);
3555 prefix(src, dst);
3556 emit_int16(0x0F, (unsigned char)0xBF);
3557 emit_operand(dst, src, 0);
3558 }
3559
3560 void Assembler::movswl(Register dst, Register src) { // movsxw
3561 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3562 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
3563 }
3564
3565 void Assembler::movups(XMMRegister dst, Address src) {
3566 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3567 InstructionMark im(this);
3568 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3569 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3570 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3571 emit_int8(0x10);
3572 emit_operand(dst, src, 0);
3573 }
3574
3575 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
3576 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
3577 InstructionMark im(this);
3578 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3579 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3580 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3581 emit_int8(0x10);
3582 emit_operand(dst, src, 0);
3583 }
3584
3585 void Assembler::movups(Address dst, XMMRegister src) {
3586 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3587 InstructionMark im(this);
3588 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3589 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3590 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3591 emit_int8(0x11);
3592 emit_operand(src, dst, 0);
3593 }
3594
3595 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
3596 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
3597 InstructionMark im(this);
3598 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3599 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3600 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3601 emit_int8(0x11);
3602 emit_operand(src, dst, 0);
3603 }
3604
3605 void Assembler::movw(Address dst, int imm16) {
3606 InstructionMark im(this);
3607
3608 emit_int8(0x66); // switch to 16-bit mode
3609 prefix(dst);
3610 emit_int8((unsigned char)0xC7);
3611 emit_operand(rax, dst, 2);
3612 emit_int16(imm16);
3613 }
3614
3615 void Assembler::movw(Register dst, Address src) {
3616 InstructionMark im(this);
3617 emit_int8(0x66);
3618 prefix(src, dst);
3619 emit_int8((unsigned char)0x8B);
3620 emit_operand(dst, src, 0);
3621 }
3622
3623 void Assembler::movw(Address dst, Register src) {
3624 InstructionMark im(this);
3625 emit_int8(0x66);
3626 prefix(dst, src);
3627 emit_int8((unsigned char)0x89);
3628 emit_operand(src, dst, 0);
3629 }
3630
3631 void Assembler::movzbl(Register dst, Address src) { // movzxb
3632 InstructionMark im(this);
3633 prefix(src, dst);
3634 emit_int16(0x0F, (unsigned char)0xB6);
3635 emit_operand(dst, src, 0);
3636 }
3637
3638 void Assembler::movzbl(Register dst, Register src) { // movzxb
3639 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3640 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3641 emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
3642 }
3643
3644 void Assembler::movzwl(Register dst, Address src) { // movzxw
3645 InstructionMark im(this);
3646 prefix(src, dst);
3647 emit_int16(0x0F, (unsigned char)0xB7);
3648 emit_operand(dst, src, 0);
3649 }
3650
3651 void Assembler::movzwl(Register dst, Register src) { // movzxw
3652 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3653 emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
3654 }
3655
3656 void Assembler::mull(Address src) {
3657 InstructionMark im(this);
3658 prefix(src);
3659 emit_int8((unsigned char)0xF7);
3660 emit_operand(rsp, src, 0);
3661 }
3662
3663 void Assembler::mull(Register src) {
3664 int encode = prefix_and_encode(src->encoding());
3665 emit_int16((unsigned char)0xF7, (0xE0 | encode));
3666 }
3667
3668 void Assembler::mulsd(XMMRegister dst, Address src) {
3669 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3670 InstructionMark im(this);
3671 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3672 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3673 attributes.set_rex_vex_w_reverted();
3674 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3675 emit_int8(0x59);
3676 emit_operand(dst, src, 0);
3677 }
3678
3679 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3680 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3681 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3682 attributes.set_rex_vex_w_reverted();
3683 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3684 emit_int16(0x59, (0xC0 | encode));
3685 }
3686
3687 void Assembler::mulss(XMMRegister dst, Address src) {
3688 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3689 InstructionMark im(this);
3690 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3691 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3692 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3693 emit_int8(0x59);
3694 emit_operand(dst, src, 0);
3695 }
3696
3697 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3698 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3699 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3700 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3701 emit_int16(0x59, (0xC0 | encode));
3702 }
3703
3704 void Assembler::negl(Register dst) {
3705 int encode = prefix_and_encode(dst->encoding());
3706 emit_int16((unsigned char)0xF7, (0xD8 | encode));
3707 }
3708
3709 void Assembler::negl(Address dst) {
3710 InstructionMark im(this);
3711 prefix(dst);
3712 emit_int8((unsigned char)0xF7);
3713 emit_operand(as_Register(3), dst, 0);
3714 }
3715
3716 void Assembler::nop(int i) {
3717 #ifdef ASSERT
3718 assert(i > 0, " ");
3719 // The fancy nops aren't currently recognized by debuggers making it a
3720 // pain to disassemble code while debugging. If asserts are on clearly
3721 // speed is not an issue so simply use the single byte traditional nop
3722 // to do alignment.
3723
3724 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3725 return;
3726
3727 #endif // ASSERT
3728
3729 if (UseAddressNop && VM_Version::is_intel()) {
3730 //
3731 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3732 // 1: 0x90
3733 // 2: 0x66 0x90
3734 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3735 // 4: 0x0F 0x1F 0x40 0x00
3736 // 5: 0x0F 0x1F 0x44 0x00 0x00
3737 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3738 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3739 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3740 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3741 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3742 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3743
3744 // The rest coding is Intel specific - don't use consecutive address nops
3745
3746 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3747 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3748 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3749 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3750
3751 while(i >= 15) {
3752 // For Intel don't generate consecutive address nops (mix with regular nops)
3753 i -= 15;
3754 emit_int24(0x66, 0x66, 0x66);
3755 addr_nop_8();
3756 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3757 }
3758 switch (i) {
3759 case 14:
3760 emit_int8(0x66); // size prefix
3761 case 13:
3762 emit_int8(0x66); // size prefix
3763 case 12:
3764 addr_nop_8();
3765 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3766 break;
3767 case 11:
3768 emit_int8(0x66); // size prefix
3769 case 10:
3770 emit_int8(0x66); // size prefix
3771 case 9:
3772 emit_int8(0x66); // size prefix
3773 case 8:
3774 addr_nop_8();
3775 break;
3776 case 7:
3777 addr_nop_7();
3778 break;
3779 case 6:
3780 emit_int8(0x66); // size prefix
3781 case 5:
3782 addr_nop_5();
3783 break;
3784 case 4:
3785 addr_nop_4();
3786 break;
3787 case 3:
3788 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3789 emit_int8(0x66); // size prefix
3790 case 2:
3791 emit_int8(0x66); // size prefix
3792 case 1:
3793 emit_int8((unsigned char)0x90);
3794 // nop
3795 break;
3796 default:
3797 assert(i == 0, " ");
3798 }
3799 return;
3800 }
3801 if (UseAddressNop && VM_Version::is_amd_family()) {
3802 //
3803 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3804 // 1: 0x90
3805 // 2: 0x66 0x90
3806 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3807 // 4: 0x0F 0x1F 0x40 0x00
3808 // 5: 0x0F 0x1F 0x44 0x00 0x00
3809 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3810 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3811 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3812 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3813 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3814 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3815
3816 // The rest coding is AMD specific - use consecutive address nops
3817
3818 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3819 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3820 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3821 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3822 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3823 // Size prefixes (0x66) are added for larger sizes
3824
3825 while(i >= 22) {
3826 i -= 11;
3827 emit_int24(0x66, 0x66, 0x66);
3828 addr_nop_8();
3829 }
3830 // Generate first nop for size between 21-12
3831 switch (i) {
3832 case 21:
3833 i -= 1;
3834 emit_int8(0x66); // size prefix
3835 case 20:
3836 case 19:
3837 i -= 1;
3838 emit_int8(0x66); // size prefix
3839 case 18:
3840 case 17:
3841 i -= 1;
3842 emit_int8(0x66); // size prefix
3843 case 16:
3844 case 15:
3845 i -= 8;
3846 addr_nop_8();
3847 break;
3848 case 14:
3849 case 13:
3850 i -= 7;
3851 addr_nop_7();
3852 break;
3853 case 12:
3854 i -= 6;
3855 emit_int8(0x66); // size prefix
3856 addr_nop_5();
3857 break;
3858 default:
3859 assert(i < 12, " ");
3860 }
3861
3862 // Generate second nop for size between 11-1
3863 switch (i) {
3864 case 11:
3865 emit_int8(0x66); // size prefix
3866 case 10:
3867 emit_int8(0x66); // size prefix
3868 case 9:
3869 emit_int8(0x66); // size prefix
3870 case 8:
3871 addr_nop_8();
3872 break;
3873 case 7:
3874 addr_nop_7();
3875 break;
3876 case 6:
3877 emit_int8(0x66); // size prefix
3878 case 5:
3879 addr_nop_5();
3880 break;
3881 case 4:
3882 addr_nop_4();
3883 break;
3884 case 3:
3885 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3886 emit_int8(0x66); // size prefix
3887 case 2:
3888 emit_int8(0x66); // size prefix
3889 case 1:
3890 emit_int8((unsigned char)0x90);
3891 // nop
3892 break;
3893 default:
3894 assert(i == 0, " ");
3895 }
3896 return;
3897 }
3898
3899 if (UseAddressNop && VM_Version::is_zx()) {
3900 //
3901 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3902 // 1: 0x90
3903 // 2: 0x66 0x90
3904 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3905 // 4: 0x0F 0x1F 0x40 0x00
3906 // 5: 0x0F 0x1F 0x44 0x00 0x00
3907 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3908 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3909 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3910 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3911 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3912 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3913
3914 // The rest coding is ZX specific - don't use consecutive address nops
3915
3916 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3917 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3918 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3919 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3920
3921 while (i >= 15) {
3922 // For ZX don't generate consecutive address nops (mix with regular nops)
3923 i -= 15;
3924 emit_int24(0x66, 0x66, 0x66);
3925 addr_nop_8();
3926 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3927 }
3928 switch (i) {
3929 case 14:
3930 emit_int8(0x66); // size prefix
3931 case 13:
3932 emit_int8(0x66); // size prefix
3933 case 12:
3934 addr_nop_8();
3935 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3936 break;
3937 case 11:
3938 emit_int8(0x66); // size prefix
3939 case 10:
3940 emit_int8(0x66); // size prefix
3941 case 9:
3942 emit_int8(0x66); // size prefix
3943 case 8:
3944 addr_nop_8();
3945 break;
3946 case 7:
3947 addr_nop_7();
3948 break;
3949 case 6:
3950 emit_int8(0x66); // size prefix
3951 case 5:
3952 addr_nop_5();
3953 break;
3954 case 4:
3955 addr_nop_4();
3956 break;
3957 case 3:
3958 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3959 emit_int8(0x66); // size prefix
3960 case 2:
3961 emit_int8(0x66); // size prefix
3962 case 1:
3963 emit_int8((unsigned char)0x90);
3964 // nop
3965 break;
3966 default:
3967 assert(i == 0, " ");
3968 }
3969 return;
3970 }
3971
3972 // Using nops with size prefixes "0x66 0x90".
3973 // From AMD Optimization Guide:
3974 // 1: 0x90
3975 // 2: 0x66 0x90
3976 // 3: 0x66 0x66 0x90
3977 // 4: 0x66 0x66 0x66 0x90
3978 // 5: 0x66 0x66 0x90 0x66 0x90
3979 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
3980 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3981 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3982 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3983 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3984 //
3985 while (i > 12) {
3986 i -= 4;
3987 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3988 }
3989 // 1 - 12 nops
3990 if (i > 8) {
3991 if (i > 9) {
3992 i -= 1;
3993 emit_int8(0x66);
3994 }
3995 i -= 3;
3996 emit_int24(0x66, 0x66, (unsigned char)0x90);
3997 }
3998 // 1 - 8 nops
3999 if (i > 4) {
4000 if (i > 6) {
4001 i -= 1;
4002 emit_int8(0x66);
4003 }
4004 i -= 3;
4005 emit_int24(0x66, 0x66, (unsigned char)0x90);
4006 }
4007 switch (i) {
4008 case 4:
4009 emit_int8(0x66);
4010 case 3:
4011 emit_int8(0x66);
4012 case 2:
4013 emit_int8(0x66);
4014 case 1:
4015 emit_int8((unsigned char)0x90);
4016 break;
4017 default:
4018 assert(i == 0, " ");
4019 }
4020 }
4021
4022 void Assembler::notl(Register dst) {
4023 int encode = prefix_and_encode(dst->encoding());
4024 emit_int16((unsigned char)0xF7, (0xD0 | encode));
4025 }
4026
4027 void Assembler::orw(Register dst, Register src) {
4028 (void)prefix_and_encode(dst->encoding(), src->encoding());
4029 emit_arith(0x0B, 0xC0, dst, src);
4030 }
4031
4032 void Assembler::orl(Address dst, int32_t imm32) {
4033 InstructionMark im(this);
4034 prefix(dst);
4035 emit_arith_operand(0x81, rcx, dst, imm32);
4036 }
4037
4038 void Assembler::orl(Register dst, int32_t imm32) {
4039 prefix(dst);
4040 emit_arith(0x81, 0xC8, dst, imm32);
4041 }
4042
4043 void Assembler::orl(Register dst, Address src) {
4044 InstructionMark im(this);
4045 prefix(src, dst);
4046 emit_int8(0x0B);
4047 emit_operand(dst, src, 0);
4048 }
4049
4050 void Assembler::orl(Register dst, Register src) {
4051 (void) prefix_and_encode(dst->encoding(), src->encoding());
4052 emit_arith(0x0B, 0xC0, dst, src);
4053 }
4054
4055 void Assembler::orl(Address dst, Register src) {
4056 InstructionMark im(this);
4057 prefix(dst, src);
4058 emit_int8(0x09);
4059 emit_operand(src, dst, 0);
4060 }
4061
4062 void Assembler::orb(Address dst, int imm8) {
4063 InstructionMark im(this);
4064 prefix(dst);
4065 emit_int8((unsigned char)0x80);
4066 emit_operand(rcx, dst, 1);
4067 emit_int8(imm8);
4068 }
4069
4070 void Assembler::orb(Address dst, Register src) {
4071 InstructionMark im(this);
4072 prefix(dst, src, true);
4073 emit_int8(0x08);
4074 emit_operand(src, dst, 0);
4075 }
4076
4077 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
4078 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4079 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4080 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4081 emit_int16(0x63, (0xC0 | encode));
4082 }
4083
4084 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4085 assert(UseAVX > 0, "some form of AVX must be enabled");
4086 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4087 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4088 emit_int16(0x63, (0xC0 | encode));
4089 }
4090
4091 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
4092 assert(VM_Version::supports_sse2(), "");
4093 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4094 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4095 emit_int16(0x6B, (0xC0 | encode));
4096 }
4097
4098 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4099 assert(UseAVX > 0, "some form of AVX must be enabled");
4100 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4101 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4102 emit_int16(0x6B, (0xC0 | encode));
4103 }
4104
4105 void Assembler::packuswb(XMMRegister dst, Address src) {
4106 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4107 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4108 InstructionMark im(this);
4109 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4110 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4111 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4112 emit_int8(0x67);
4113 emit_operand(dst, src, 0);
4114 }
4115
4116 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
4117 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4118 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4119 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4120 emit_int16(0x67, (0xC0 | encode));
4121 }
4122
4123 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4124 assert(UseAVX > 0, "some form of AVX must be enabled");
4125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4126 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4127 emit_int16(0x67, (0xC0 | encode));
4128 }
4129
4130 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
4131 assert(VM_Version::supports_sse4_1(), "");
4132 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4133 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4134 emit_int16(0x2B, (0xC0 | encode));
4135 }
4136
4137 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4138 assert(UseAVX > 0, "some form of AVX must be enabled");
4139 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4140 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4141 emit_int16(0x2B, (0xC0 | encode));
4142 }
4143
4144 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4145 assert(VM_Version::supports_avx2(), "");
4146 assert(vector_len != AVX_128bit, "");
4147 // VEX.256.66.0F3A.W1 00 /r ib
4148 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4149 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4150 emit_int24(0x00, (0xC0 | encode), imm8);
4151 }
4152
4153 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4154 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
4155 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported");
4156 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4157 attributes.set_is_evex_instruction();
4158 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4159 emit_int16(0x36, (0xC0 | encode));
4160 }
4161
4162 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4163 assert(VM_Version::supports_avx512_vbmi(), "");
4164 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4165 attributes.set_is_evex_instruction();
4166 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4167 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4168 }
4169
4170 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4171 assert(VM_Version::supports_avx512_vbmi(), "");
4172 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4173 attributes.set_is_evex_instruction();
4174 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4175 emit_int8((unsigned char)0x8D);
4176 emit_operand(dst, src, 0);
4177 }
4178
4179 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4180 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
4181 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
4182 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
4183 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4184 attributes.set_is_evex_instruction();
4185 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4186 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4187 }
4188
4189 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4190 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4191 // VEX.NDS.256.66.0F38.W0 36 /r
4192 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4193 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4194 emit_int16(0x36, (0xC0 | encode));
4195 }
4196
4197 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4198 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4199 // VEX.NDS.256.66.0F38.W0 36 /r
4200 InstructionMark im(this);
4201 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4202 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4203 emit_int8(0x36);
4204 emit_operand(dst, src, 0);
4205 }
4206
4207 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4208 assert(VM_Version::supports_avx2(), "");
4209 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4210 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4211 emit_int24(0x46, (0xC0 | encode), imm8);
4212 }
4213
4214 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4215 assert(VM_Version::supports_avx(), "");
4216 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4217 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4218 emit_int24(0x06, (0xC0 | encode), imm8);
4219 }
4220
4221 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4222 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4223 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4224 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4225 emit_int24(0x04, (0xC0 | encode), imm8);
4226 }
4227
4228 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4229 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4230 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
4231 attributes.set_rex_vex_w_reverted();
4232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4233 emit_int24(0x05, (0xC0 | encode), imm8);
4234 }
4235
4236 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4237 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4238 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
4239 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4240 emit_int24(0x01, (0xC0 | encode), imm8);
4241 }
4242
4243 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4244 assert(VM_Version::supports_evex(), "");
4245 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4246 attributes.set_is_evex_instruction();
4247 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4248 emit_int16(0x76, (0xC0 | encode));
4249 }
4250
4251 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4252 assert(VM_Version::supports_avx512_vbmi(), "");
4253 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4254 attributes.set_is_evex_instruction();
4255 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4256 emit_int16(0x7D, (0xC0 | encode));
4257 }
4258
4259 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
4260 assert(VM_Version::supports_avx512_vbmi(), "");
4261 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4262 attributes.set_is_evex_instruction();
4263 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4264 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
4265 }
4266
4267 void Assembler::pause() {
4268 emit_int16((unsigned char)0xF3, (unsigned char)0x90);
4269 }
4270
4271 void Assembler::ud2() {
4272 emit_int16(0x0F, 0x0B);
4273 }
4274
4275 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
4276 assert(VM_Version::supports_sse4_2(), "");
4277 InstructionMark im(this);
4278 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4279 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4280 emit_int8(0x61);
4281 emit_operand(dst, src, 1);
4282 emit_int8(imm8);
4283 }
4284
4285 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
4286 assert(VM_Version::supports_sse4_2(), "");
4287 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4288 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4289 emit_int24(0x61, (0xC0 | encode), imm8);
4290 }
4291
4292 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4293 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
4294 assert(VM_Version::supports_sse2(), "");
4295 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4296 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4297 emit_int16(0x74, (0xC0 | encode));
4298 }
4299
4300 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4301 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4302 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4303 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4304 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4305 emit_int16(cond_encoding, (0xC0 | encode));
4306 }
4307
4308 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4309 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4310 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4311 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4312 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4313 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4314 emit_int16(0x74, (0xC0 | encode));
4315 }
4316
4317 // In this context, kdst is written the mask used to process the equal components
4318 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4319 assert(VM_Version::supports_avx512bw(), "");
4320 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4321 attributes.set_is_evex_instruction();
4322 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4323 emit_int16(0x74, (0xC0 | encode));
4324 }
4325
4326 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4327 assert(VM_Version::supports_avx512vlbw(), "");
4328 InstructionMark im(this);
4329 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4330 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4331 attributes.set_is_evex_instruction();
4332 int dst_enc = kdst->encoding();
4333 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4334 emit_int8(0x64);
4335 emit_operand(as_Register(dst_enc), src, 0);
4336 }
4337
4338 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4339 assert(VM_Version::supports_avx512vlbw(), "");
4340 InstructionMark im(this);
4341 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4342 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4343 attributes.reset_is_clear_context();
4344 attributes.set_embedded_opmask_register_specifier(mask);
4345 attributes.set_is_evex_instruction();
4346 int dst_enc = kdst->encoding();
4347 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4348 emit_int8(0x64);
4349 emit_operand(as_Register(dst_enc), src, 0);
4350 }
4351
4352 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4353 assert(VM_Version::supports_avx512vlbw(), "");
4354 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4355 attributes.set_is_evex_instruction();
4356 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4357 emit_int24(0x3E, (0xC0 | encode), vcc);
4358 }
4359
4360 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
4361 assert(VM_Version::supports_avx512vlbw(), "");
4362 InstructionMark im(this);
4363 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4364 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4365 attributes.set_is_evex_instruction();
4366 int dst_enc = kdst->encoding();
4367 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4368 emit_int8(0x3E);
4369 emit_operand(as_Register(dst_enc), src, 1);
4370 emit_int8(vcc);
4371 }
4372
4373 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4374 assert(VM_Version::supports_avx512bw(), "");
4375 InstructionMark im(this);
4376 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4377 attributes.set_is_evex_instruction();
4378 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4379 int dst_enc = kdst->encoding();
4380 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4381 emit_int8(0x74);
4382 emit_operand(as_Register(dst_enc), src, 0);
4383 }
4384
4385 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4386 assert(VM_Version::supports_avx512vlbw(), "");
4387 InstructionMark im(this);
4388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4389 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4390 attributes.reset_is_clear_context();
4391 attributes.set_embedded_opmask_register_specifier(mask);
4392 attributes.set_is_evex_instruction();
4393 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4394 emit_int8(0x74);
4395 emit_operand(as_Register(kdst->encoding()), src, 0);
4396 }
4397
4398 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4399 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
4400 assert(VM_Version::supports_sse2(), "");
4401 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4402 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4403 emit_int16(0x75, (0xC0 | encode));
4404 }
4405
4406 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4407 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4408 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4409 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4410 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4411 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4412 emit_int16(0x75, (0xC0 | encode));
4413 }
4414
4415 // In this context, kdst is written the mask used to process the equal components
4416 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4417 assert(VM_Version::supports_avx512bw(), "");
4418 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4419 attributes.set_is_evex_instruction();
4420 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4421 emit_int16(0x75, (0xC0 | encode));
4422 }
4423
4424 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4425 assert(VM_Version::supports_avx512bw(), "");
4426 InstructionMark im(this);
4427 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4428 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4429 attributes.set_is_evex_instruction();
4430 int dst_enc = kdst->encoding();
4431 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4432 emit_int8(0x75);
4433 emit_operand(as_Register(dst_enc), src, 0);
4434 }
4435
4436 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4437 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
4438 assert(VM_Version::supports_sse2(), "");
4439 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4440 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4441 emit_int16(0x76, (0xC0 | encode));
4442 }
4443
4444 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4445 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4446 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4447 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4448 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4449 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4450 emit_int16(0x76, (0xC0 | encode));
4451 }
4452
4453 // In this context, kdst is written the mask used to process the equal components
4454 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4455 assert(VM_Version::supports_evex(), "");
4456 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4457 attributes.set_is_evex_instruction();
4458 attributes.reset_is_clear_context();
4459 attributes.set_embedded_opmask_register_specifier(mask);
4460 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4461 emit_int16(0x76, (0xC0 | encode));
4462 }
4463
4464 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4465 assert(VM_Version::supports_evex(), "");
4466 InstructionMark im(this);
4467 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4468 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4469 attributes.set_is_evex_instruction();
4470 attributes.reset_is_clear_context();
4471 attributes.set_embedded_opmask_register_specifier(mask);
4472 int dst_enc = kdst->encoding();
4473 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4474 emit_int8(0x76);
4475 emit_operand(as_Register(dst_enc), src, 0);
4476 }
4477
4478 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4479 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
4480 assert(VM_Version::supports_sse4_1(), "");
4481 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4482 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4483 emit_int16(0x29, (0xC0 | encode));
4484 }
4485
4486 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4487 assert(VM_Version::supports_evex(), "");
4488 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4489 attributes.set_is_evex_instruction();
4490 attributes.reset_is_clear_context();
4491 attributes.set_embedded_opmask_register_specifier(mask);
4492 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4493 emit_int16(0x29, (0xC0 | encode));
4494 }
4495
4496 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4497 assert(VM_Version::supports_avx(), "");
4498 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4499 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4500 emit_int16(cond_encoding, (0xC0 | encode));
4501 }
4502
4503 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4504 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4505 assert(VM_Version::supports_avx(), "");
4506 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4507 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4508 emit_int16(0x29, (0xC0 | encode));
4509 }
4510
4511 // In this context, kdst is written the mask used to process the equal components
4512 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4513 assert(VM_Version::supports_evex(), "");
4514 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4515 attributes.reset_is_clear_context();
4516 attributes.set_is_evex_instruction();
4517 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4518 emit_int16(0x29, (0xC0 | encode));
4519 }
4520
4521 // In this context, kdst is written the mask used to process the equal components
4522 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4523 assert(VM_Version::supports_evex(), "");
4524 InstructionMark im(this);
4525 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4526 attributes.reset_is_clear_context();
4527 attributes.set_is_evex_instruction();
4528 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4529 int dst_enc = kdst->encoding();
4530 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4531 emit_int8(0x29);
4532 emit_operand(as_Register(dst_enc), src, 0);
4533 }
4534
4535 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
4536 assert(VM_Version::supports_sse4_1(), "");
4537 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4538 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4539 emit_int16(0x37, (0xC0 | encode));
4540 }
4541
4542 void Assembler::pmovmskb(Register dst, XMMRegister src) {
4543 assert(VM_Version::supports_sse2(), "");
4544 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4545 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4546 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4547 }
4548
4549 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
4550 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
4551 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), "");
4552 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4553 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4554 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4555 }
4556
4557 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
4558 assert(VM_Version::supports_avx(), "");
4559 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4560 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4561 emit_int16(0x50, (0xC0 | encode));
4562 }
4563
4564 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
4565 assert(VM_Version::supports_avx(), "");
4566 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4567 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4568 emit_int16(0x50, (0xC0 | encode));
4569 }
4570
4571
4572 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
4573 assert(VM_Version::supports_sse4_1(), "");
4574 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4575 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4576 emit_int24(0x16, (0xC0 | encode), imm8);
4577 }
4578
4579 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
4580 assert(VM_Version::supports_sse4_1(), "");
4581 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4582 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4583 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4584 emit_int8(0x16);
4585 emit_operand(src, dst, 1);
4586 emit_int8(imm8);
4587 }
4588
4589 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
4590 assert(VM_Version::supports_sse4_1(), "");
4591 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4592 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4593 emit_int24(0x16, (0xC0 | encode), imm8);
4594 }
4595
4596 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
4597 assert(VM_Version::supports_sse4_1(), "");
4598 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4599 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4600 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4601 emit_int8(0x16);
4602 emit_operand(src, dst, 1);
4603 emit_int8(imm8);
4604 }
4605
4606 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
4607 assert(VM_Version::supports_sse2(), "");
4608 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4609 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4610 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
4611 }
4612
4613 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
4614 assert(VM_Version::supports_sse4_1(), "");
4615 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4616 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4617 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4618 emit_int8(0x15);
4619 emit_operand(src, dst, 1);
4620 emit_int8(imm8);
4621 }
4622
4623 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
4624 assert(VM_Version::supports_sse4_1(), "");
4625 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4626 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4627 emit_int24(0x14, (0xC0 | encode), imm8);
4628 }
4629
4630 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
4631 assert(VM_Version::supports_sse4_1(), "");
4632 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4633 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4634 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4635 emit_int8(0x14);
4636 emit_operand(src, dst, 1);
4637 emit_int8(imm8);
4638 }
4639
4640 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
4641 assert(VM_Version::supports_sse4_1(), "");
4642 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4643 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4644 emit_int24(0x22, (0xC0 | encode), imm8);
4645 }
4646
4647 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
4648 assert(VM_Version::supports_sse4_1(), "");
4649 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4650 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4651 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4652 emit_int8(0x22);
4653 emit_operand(dst, src, 1);
4654 emit_int8(imm8);
4655 }
4656
4657 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4658 assert(VM_Version::supports_avx(), "");
4659 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4660 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4661 emit_int24(0x22, (0xC0 | encode), imm8);
4662 }
4663
4664 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
4665 assert(VM_Version::supports_sse4_1(), "");
4666 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4667 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4668 emit_int24(0x22, (0xC0 | encode), imm8);
4669 }
4670
4671 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
4672 assert(VM_Version::supports_sse4_1(), "");
4673 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4674 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4675 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4676 emit_int8(0x22);
4677 emit_operand(dst, src, 1);
4678 emit_int8(imm8);
4679 }
4680
4681 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4682 assert(VM_Version::supports_avx(), "");
4683 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4685 emit_int24(0x22, (0xC0 | encode), imm8);
4686 }
4687
4688 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
4689 assert(VM_Version::supports_sse2(), "");
4690 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4691 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4692 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4693 }
4694
4695 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
4696 assert(VM_Version::supports_sse2(), "");
4697 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4698 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4699 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4700 emit_int8((unsigned char)0xC4);
4701 emit_operand(dst, src, 1);
4702 emit_int8(imm8);
4703 }
4704
4705 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4706 assert(VM_Version::supports_avx(), "");
4707 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4708 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4709 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4710 }
4711
4712 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
4713 assert(VM_Version::supports_sse4_1(), "");
4714 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4715 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4716 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4717 emit_int8(0x20);
4718 emit_operand(dst, src, 1);
4719 emit_int8(imm8);
4720 }
4721
4722 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
4723 assert(VM_Version::supports_sse4_1(), "");
4724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4725 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4726 emit_int24(0x20, (0xC0 | encode), imm8);
4727 }
4728
4729 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4730 assert(VM_Version::supports_avx(), "");
4731 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4732 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4733 emit_int24(0x20, (0xC0 | encode), imm8);
4734 }
4735
4736 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
4737 assert(VM_Version::supports_sse4_1(), "");
4738 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4739 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4740 emit_int24(0x21, (0xC0 | encode), imm8);
4741 }
4742
4743 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4744 assert(VM_Version::supports_avx(), "");
4745 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4746 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4747 emit_int24(0x21, (0xC0 | encode), imm8);
4748 }
4749
4750 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
4751 assert(VM_Version::supports_sse4_1(), "");
4752 InstructionMark im(this);
4753 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4754 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4755 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4756 emit_int8(0x30);
4757 emit_operand(dst, src, 0);
4758 }
4759
4760 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
4761 assert(VM_Version::supports_sse4_1(), "");
4762 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4763 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4764 emit_int16(0x30, (0xC0 | encode));
4765 }
4766
4767 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
4768 assert(VM_Version::supports_sse4_1(), "");
4769 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4770 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4771 emit_int16(0x20, (0xC0 | encode));
4772 }
4773
4774 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
4775 assert(VM_Version::supports_sse4_1(), "");
4776 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4777 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4778 emit_int16(0x35, (0xC0 | encode));
4779 }
4780
4781 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
4782 assert(VM_Version::supports_sse4_1(), "");
4783 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4784 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4785 emit_int16(0x21, (0xC0 | encode));
4786 }
4787
4788 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
4789 assert(VM_Version::supports_sse4_1(), "");
4790 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4791 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4792 emit_int16(0x31, (0xC0 | encode));
4793 }
4794
4795 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
4796 assert(VM_Version::supports_sse4_1(), "");
4797 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4798 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4799 emit_int16(0x22, (0xC0 | encode));
4800 }
4801
4802 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
4803 assert(VM_Version::supports_sse4_1(), "");
4804 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4805 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4806 emit_int16(0x23, (0xC0 | encode));
4807 }
4808
4809 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4810 assert(VM_Version::supports_avx(), "");
4811 InstructionMark im(this);
4812 assert(dst != xnoreg, "sanity");
4813 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4814 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4815 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4816 emit_int8(0x30);
4817 emit_operand(dst, src, 0);
4818 }
4819
4820 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4821 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4822 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4823 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4824 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4825 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4826 emit_int16(0x30, (unsigned char) (0xC0 | encode));
4827 }
4828
4829 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4830 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4831 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4832 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4833 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4834 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4835 emit_int16(0x20, (0xC0 | encode));
4836 }
4837
4838 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
4839 assert(VM_Version::supports_avx512vlbw(), "");
4840 assert(dst != xnoreg, "sanity");
4841 InstructionMark im(this);
4842 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4843 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4844 attributes.set_embedded_opmask_register_specifier(mask);
4845 attributes.set_is_evex_instruction();
4846 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4847 emit_int8(0x30);
4848 emit_operand(dst, src, 0);
4849 }
4850
4851 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
4852 assert(VM_Version::supports_evex(), "");
4853 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
4854 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4855 attributes.set_is_evex_instruction();
4856 attributes.set_embedded_opmask_register_specifier(mask);
4857 if (merge) {
4858 attributes.reset_is_clear_context();
4859 }
4860 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4861 emit_int16((unsigned char)0xDB, (0xC0 | encode));
4862 }
4863
4864 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4865 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4866 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4867 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4868 emit_int16(0x35, (0xC0 | encode));
4869 }
4870
4871 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4872 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4873 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4874 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4875 emit_int16(0x31, (0xC0 | encode));
4876 }
4877
4878 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4879 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4880 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4881 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4882 emit_int16(0x32, (0xC0 | encode));
4883 }
4884
4885 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4886 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4887 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4888 VM_Version::supports_evex(), "");
4889 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4890 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4891 emit_int16(0x21, (0xC0 | encode));
4892 }
4893
4894 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4895 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4896 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4897 VM_Version::supports_evex(), "");
4898 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4899 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4900 emit_int16(0x22, (0xC0 | encode));
4901 }
4902
4903 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
4904 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4905 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4906 VM_Version::supports_evex(), "");
4907 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4908 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4909 emit_int16(0x23, (0xC0 | encode));
4910 }
4911
4912 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4913 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4914 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4915 VM_Version::supports_evex(), "");
4916 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4917 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4918 emit_int16(0x24, (0xC0 | encode));
4919 }
4920
4921 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4922 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4923 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4924 VM_Version::supports_evex(), "");
4925 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4926 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4927 emit_int16(0x25, (0xC0 | encode));
4928 }
4929
4930 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
4931 assert(VM_Version::supports_avx512vlbw(), "");
4932 assert(src != xnoreg, "sanity");
4933 InstructionMark im(this);
4934 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4935 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4936 attributes.set_is_evex_instruction();
4937 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4938 emit_int8(0x30);
4939 emit_operand(src, dst, 0);
4940 }
4941
4942 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
4943 assert(VM_Version::supports_avx512vlbw(), "");
4944 assert(src != xnoreg, "sanity");
4945 InstructionMark im(this);
4946 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4947 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4948 attributes.reset_is_clear_context();
4949 attributes.set_embedded_opmask_register_specifier(mask);
4950 attributes.set_is_evex_instruction();
4951 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4952 emit_int8(0x30);
4953 emit_operand(src, dst, 0);
4954 }
4955
4956 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
4957 assert(VM_Version::supports_evex(), "");
4958 assert(src != xnoreg, "sanity");
4959 InstructionMark im(this);
4960 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4961 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
4962 attributes.set_is_evex_instruction();
4963 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4964 emit_int8(0x31);
4965 emit_operand(src, dst, 0);
4966 }
4967
4968 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
4969 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4970 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4971 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4972 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4973 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4974 emit_int16(0x33, (0xC0 | encode));
4975 }
4976
4977 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4978 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4979 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4980 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4981 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4982 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4983 emit_int16(0x34, (0xC0 | encode));
4984 }
4985
4986 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
4987 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4988 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4989 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4990 emit_int16((unsigned char)0xF5, (0xC0 | encode));
4991 }
4992
4993 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4994 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4995 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4996 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
4997 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4998 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4999 emit_int16((unsigned char)0xF5, (0xC0 | encode));
5000 }
5001
5002 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5003 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5004 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5005 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5006 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5007 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5008 emit_int16(0x04, (0xC0 | encode));
5009 }
5010
5011 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5012 assert(VM_Version::supports_evex(), "");
5013 assert(VM_Version::supports_avx512_vnni(), "must support vnni");
5014 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5015 attributes.set_is_evex_instruction();
5016 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5017 emit_int16(0x52, (0xC0 | encode));
5018 }
5019
5020 // generic
5021 void Assembler::pop(Register dst) {
5022 int encode = prefix_and_encode(dst->encoding());
5023 emit_int8(0x58 | encode);
5024 }
5025
5026 void Assembler::popcntl(Register dst, Address src) {
5027 assert(VM_Version::supports_popcnt(), "must support");
5028 InstructionMark im(this);
5029 emit_int8((unsigned char)0xF3);
5030 prefix(src, dst);
5031 emit_int16(0x0F, (unsigned char)0xB8);
5032 emit_operand(dst, src, 0);
5033 }
5034
5035 void Assembler::popcntl(Register dst, Register src) {
5036 assert(VM_Version::supports_popcnt(), "must support");
5037 emit_int8((unsigned char)0xF3);
5038 int encode = prefix_and_encode(dst->encoding(), src->encoding());
5039 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
5040 }
5041
5042 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5043 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5044 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5045 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5046 attributes.set_embedded_opmask_register_specifier(mask);
5047 attributes.set_is_evex_instruction();
5048 if (merge) {
5049 attributes.reset_is_clear_context();
5050 }
5051 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5052 emit_int16(0x54, (0xC0 | encode));
5053 }
5054
5055 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5056 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5057 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5058 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5059 attributes.set_is_evex_instruction();
5060 attributes.set_embedded_opmask_register_specifier(mask);
5061 if (merge) {
5062 attributes.reset_is_clear_context();
5063 }
5064 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5065 emit_int16(0x54, (0xC0 | encode));
5066 }
5067
5068 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5069 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5070 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5071 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5072 attributes.set_is_evex_instruction();
5073 attributes.set_embedded_opmask_register_specifier(mask);
5074 if (merge) {
5075 attributes.reset_is_clear_context();
5076 }
5077 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5078 emit_int16(0x55, (0xC0 | encode));
5079 }
5080
5081 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5082 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5083 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5084 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5085 attributes.set_is_evex_instruction();
5086 attributes.set_embedded_opmask_register_specifier(mask);
5087 if (merge) {
5088 attributes.reset_is_clear_context();
5089 }
5090 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5091 emit_int16(0x55, (0xC0 | encode));
5092 }
5093
5094 void Assembler::popf() {
5095 emit_int8((unsigned char)0x9D);
5096 }
5097
5098 #ifndef _LP64 // no 32bit push/pop on amd64
5099 void Assembler::popl(Address dst) {
5100 // NOTE: this will adjust stack by 8byte on 64bits
5101 InstructionMark im(this);
5102 prefix(dst);
5103 emit_int8((unsigned char)0x8F);
5104 emit_operand(rax, dst, 0);
5105 }
5106 #endif
5107
5108 void Assembler::prefetchnta(Address src) {
5109 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5110 InstructionMark im(this);
5111 prefix(src);
5112 emit_int16(0x0F, 0x18);
5113 emit_operand(rax, src, 0); // 0, src
5114 }
5115
5116 void Assembler::prefetchr(Address src) {
5117 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5118 InstructionMark im(this);
5119 prefix(src);
5120 emit_int16(0x0F, 0x0D);
5121 emit_operand(rax, src, 0); // 0, src
5122 }
5123
5124 void Assembler::prefetcht0(Address src) {
5125 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5126 InstructionMark im(this);
5127 prefix(src);
5128 emit_int16(0x0F, 0x18);
5129 emit_operand(rcx, src, 0); // 1, src
5130 }
5131
5132 void Assembler::prefetcht1(Address src) {
5133 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5134 InstructionMark im(this);
5135 prefix(src);
5136 emit_int16(0x0F, 0x18);
5137 emit_operand(rdx, src, 0); // 2, src
5138 }
5139
5140 void Assembler::prefetcht2(Address src) {
5141 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5142 InstructionMark im(this);
5143 prefix(src);
5144 emit_int16(0x0F, 0x18);
5145 emit_operand(rbx, src, 0); // 3, src
5146 }
5147
5148 void Assembler::prefetchw(Address src) {
5149 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5150 InstructionMark im(this);
5151 prefix(src);
5152 emit_int16(0x0F, 0x0D);
5153 emit_operand(rcx, src, 0); // 1, src
5154 }
5155
5156 void Assembler::prefix(Prefix p) {
5157 emit_int8(p);
5158 }
5159
5160 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
5161 assert(VM_Version::supports_ssse3(), "");
5162 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5163 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5164 emit_int16(0x00, (0xC0 | encode));
5165 }
5166
5167 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
5168 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
5169 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5170 attributes.set_is_evex_instruction();
5171 attributes.set_embedded_opmask_register_specifier(mask);
5172 if (merge) {
5173 attributes.reset_is_clear_context();
5174 }
5175 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5176 emit_int16(0x00, (0xC0 | encode));
5177 }
5178
5179 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5180 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5181 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5182 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5183 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5184 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5185 emit_int16(0x00, (0xC0 | encode));
5186 }
5187
5188 void Assembler::pshufb(XMMRegister dst, Address src) {
5189 assert(VM_Version::supports_ssse3(), "");
5190 InstructionMark im(this);
5191 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5192 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5193 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5194 emit_int8(0x00);
5195 emit_operand(dst, src, 0);
5196 }
5197
5198 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
5199 assert(isByte(mode), "invalid value");
5200 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5201 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
5202 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5203 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5204 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5205 }
5206
5207 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5208 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5209 (vector_len == AVX_256bit? VM_Version::supports_avx2() :
5210 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
5211 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5212 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5213 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5214 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5215 }
5216
5217 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
5218 assert(isByte(mode), "invalid value");
5219 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5220 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5221 InstructionMark im(this);
5222 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5223 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5224 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5225 emit_int8(0x70);
5226 emit_operand(dst, src, 1);
5227 emit_int8(mode & 0xFF);
5228 }
5229
5230 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
5231 assert(isByte(mode), "invalid value");
5232 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5233 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5234 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5235 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5236 }
5237
5238 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5239 assert(isByte(mode), "invalid value");
5240 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5241 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5242 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5243 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5244 }
5245
5246 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
5247 assert(isByte(mode), "invalid value");
5248 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5249 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5250 InstructionMark im(this);
5251 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5252 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5253 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5254 emit_int8(0x70);
5255 emit_operand(dst, src, 1);
5256 emit_int8(mode & 0xFF);
5257 }
5258
5259 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5260 assert(VM_Version::supports_evex(), "requires EVEX support");
5261 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
5262 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5263 attributes.set_is_evex_instruction();
5264 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5265 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
5266 }
5267
5268 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
5269 assert(isByte(imm8), "invalid value");
5270 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5271 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5272 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5273 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5274 }
5275
5276 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5277 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5278 attributes.set_rex_vex_w_reverted();
5279 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5280 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5281 }
5282
5283 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
5284 assert(isByte(imm8), "invalid value");
5285 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5286 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5287 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5288 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5289 }
5290
5291 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5292 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5293 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5294 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5295 }
5296
5297 void Assembler::psrldq(XMMRegister dst, int shift) {
5298 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5299 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5301 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5302 emit_int24(0x73, (0xC0 | encode), shift);
5303 }
5304
5305 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5306 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5307 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5308 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5309 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5310 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5311 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5312 }
5313
5314 void Assembler::pslldq(XMMRegister dst, int shift) {
5315 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5316 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5317 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5318 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
5319 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5320 emit_int24(0x73, (0xC0 | encode), shift);
5321 }
5322
5323 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5324 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5325 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5326 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5327 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5328 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5329 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5330 }
5331
5332 void Assembler::ptest(XMMRegister dst, Address src) {
5333 assert(VM_Version::supports_sse4_1(), "");
5334 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5335 InstructionMark im(this);
5336 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5337 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5338 emit_int8(0x17);
5339 emit_operand(dst, src, 0);
5340 }
5341
5342 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
5343 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
5344 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5345 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5346 emit_int8(0x17);
5347 emit_int8((0xC0 | encode));
5348 }
5349
5350 void Assembler::vptest(XMMRegister dst, Address src) {
5351 assert(VM_Version::supports_avx(), "");
5352 InstructionMark im(this);
5353 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5354 assert(dst != xnoreg, "sanity");
5355 // swap src<->dst for encoding
5356 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5357 emit_int8(0x17);
5358 emit_operand(dst, src, 0);
5359 }
5360
5361 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
5362 assert(VM_Version::supports_avx(), "");
5363 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5364 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5365 emit_int16(0x17, (0xC0 | encode));
5366 }
5367
5368 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
5369 assert(VM_Version::supports_avx(), "");
5370 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5371 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5372 emit_int16(0x17, (0xC0 | encode));
5373 }
5374
5375 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5376 assert(VM_Version::supports_avx512vlbw(), "");
5377 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
5378 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5379 attributes.set_is_evex_instruction();
5380 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5381 emit_int16((unsigned char)0x26, (0xC0 | encode));
5382 }
5383
5384 void Assembler::punpcklbw(XMMRegister dst, Address src) {
5385 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5386 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5387 InstructionMark im(this);
5388 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5389 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5390 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5391 emit_int8(0x60);
5392 emit_operand(dst, src, 0);
5393 }
5394
5395 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
5396 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5397 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5398 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5399 emit_int16(0x60, (0xC0 | encode));
5400 }
5401
5402 void Assembler::punpckldq(XMMRegister dst, Address src) {
5403 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5404 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5405 InstructionMark im(this);
5406 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5407 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5408 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5409 emit_int8(0x62);
5410 emit_operand(dst, src, 0);
5411 }
5412
5413 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
5414 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5415 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5416 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5417 emit_int16(0x62, (0xC0 | encode));
5418 }
5419
5420 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
5421 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5422 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5423 attributes.set_rex_vex_w_reverted();
5424 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5425 emit_int16(0x6C, (0xC0 | encode));
5426 }
5427
5428 void Assembler::push(int32_t imm32) {
5429 // in 64bits we push 64bits onto the stack but only
5430 // take a 32bit immediate
5431 emit_int8(0x68);
5432 emit_int32(imm32);
5433 }
5434
5435 void Assembler::push(Register src) {
5436 int encode = prefix_and_encode(src->encoding());
5437 emit_int8(0x50 | encode);
5438 }
5439
5440 void Assembler::pushf() {
5441 emit_int8((unsigned char)0x9C);
5442 }
5443
5444 #ifndef _LP64 // no 32bit push/pop on amd64
5445 void Assembler::pushl(Address src) {
5446 // Note this will push 64bit on 64bit
5447 InstructionMark im(this);
5448 prefix(src);
5449 emit_int8((unsigned char)0xFF);
5450 emit_operand(rsi, src, 0);
5451 }
5452 #endif
5453
5454 void Assembler::rcll(Register dst, int imm8) {
5455 assert(isShiftCount(imm8), "illegal shift count");
5456 int encode = prefix_and_encode(dst->encoding());
5457 if (imm8 == 1) {
5458 emit_int16((unsigned char)0xD1, (0xD0 | encode));
5459 } else {
5460 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
5461 }
5462 }
5463
5464 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
5465 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5466 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5467 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5468 emit_int16(0x53, (0xC0 | encode));
5469 }
5470
5471 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
5472 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5473 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5474 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5475 emit_int16(0x53, (0xC0 | encode));
5476 }
5477
5478 void Assembler::rdtsc() {
5479 emit_int16(0x0F, 0x31);
5480 }
5481
5482 void Assembler::rdtscp() {
5483 emit_int24(0x0F, 0x01, (unsigned char)0xF9);
5484 }
5485
5486 // copies data from [esi] to [edi] using rcx pointer sized words
5487 // generic
5488 void Assembler::rep_mov() {
5489 // REP
5490 // MOVSQ
5491 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
5492 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xA5);)
5493 }
5494
5495 // sets rcx bytes with rax, value at [edi]
5496 void Assembler::rep_stosb() {
5497 // REP
5498 // STOSB
5499 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
5500 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAA);)
5501 }
5502
5503 // sets rcx pointer sized words with rax, value at [edi]
5504 // generic
5505 void Assembler::rep_stos() {
5506 // REP
5507 // LP64:STOSQ, LP32:STOSD
5508 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
5509 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAB);)
5510 }
5511
5512 // scans rcx pointer sized words at [edi] for occurrence of rax,
5513 // generic
5514 void Assembler::repne_scan() { // repne_scan
5515 // SCASQ
5516 LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
5517 NOT_LP64( emit_int16((unsigned char)0xF2, (unsigned char)0xAF);)
5518 }
5519
5520 #ifdef _LP64
5521 // scans rcx 4 byte words at [edi] for occurrence of rax,
5522 // generic
5523 void Assembler::repne_scanl() { // repne_scan
5524 // SCASL
5525 emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
5526 }
5527 #endif
5528
5529 void Assembler::ret(int imm16) {
5530 if (imm16 == 0) {
5531 emit_int8((unsigned char)0xC3);
5532 } else {
5533 emit_int8((unsigned char)0xC2);
5534 emit_int16(imm16);
5535 }
5536 }
5537
5538 void Assembler::roll(Register dst, int imm8) {
5539 assert(isShiftCount(imm8), "illegal shift count");
5540 int encode = prefix_and_encode(dst->encoding());
5541 if (imm8 == 1) {
5542 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5543 } else {
5544 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5545 }
5546 }
5547
5548 void Assembler::roll(Register dst) {
5549 int encode = prefix_and_encode(dst->encoding());
5550 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5551 }
5552
5553 void Assembler::rorl(Register dst, int imm8) {
5554 assert(isShiftCount(imm8), "illegal shift count");
5555 int encode = prefix_and_encode(dst->encoding());
5556 if (imm8 == 1) {
5557 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5558 } else {
5559 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5560 }
5561 }
5562
5563 void Assembler::rorl(Register dst) {
5564 int encode = prefix_and_encode(dst->encoding());
5565 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5566 }
5567
5568 #ifdef _LP64
5569 void Assembler::rorq(Register dst) {
5570 int encode = prefixq_and_encode(dst->encoding());
5571 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5572 }
5573
5574 void Assembler::rorq(Register dst, int imm8) {
5575 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5576 int encode = prefixq_and_encode(dst->encoding());
5577 if (imm8 == 1) {
5578 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5579 } else {
5580 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5581 }
5582 }
5583
5584 void Assembler::rolq(Register dst) {
5585 int encode = prefixq_and_encode(dst->encoding());
5586 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5587 }
5588
5589 void Assembler::rolq(Register dst, int imm8) {
5590 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5591 int encode = prefixq_and_encode(dst->encoding());
5592 if (imm8 == 1) {
5593 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5594 } else {
5595 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5596 }
5597 }
5598 #endif
5599
5600 void Assembler::sahf() {
5601 #ifdef _LP64
5602 // Not supported in 64bit mode
5603 ShouldNotReachHere();
5604 #endif
5605 emit_int8((unsigned char)0x9E);
5606 }
5607
5608 void Assembler::sall(Address dst, int imm8) {
5609 InstructionMark im(this);
5610 assert(isShiftCount(imm8), "illegal shift count");
5611 prefix(dst);
5612 if (imm8 == 1) {
5613 emit_int8((unsigned char)0xD1);
5614 emit_operand(as_Register(4), dst, 0);
5615 }
5616 else {
5617 emit_int8((unsigned char)0xC1);
5618 emit_operand(as_Register(4), dst, 1);
5619 emit_int8(imm8);
5620 }
5621 }
5622
5623 void Assembler::sall(Address dst) {
5624 InstructionMark im(this);
5625 prefix(dst);
5626 emit_int8((unsigned char)0xD3);
5627 emit_operand(as_Register(4), dst, 0);
5628 }
5629
5630 void Assembler::sall(Register dst, int imm8) {
5631 assert(isShiftCount(imm8), "illegal shift count");
5632 int encode = prefix_and_encode(dst->encoding());
5633 if (imm8 == 1) {
5634 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5635 } else {
5636 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5637 }
5638 }
5639
5640 void Assembler::sall(Register dst) {
5641 int encode = prefix_and_encode(dst->encoding());
5642 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5643 }
5644
5645 void Assembler::sarl(Address dst, int imm8) {
5646 assert(isShiftCount(imm8), "illegal shift count");
5647 InstructionMark im(this);
5648 prefix(dst);
5649 if (imm8 == 1) {
5650 emit_int8((unsigned char)0xD1);
5651 emit_operand(as_Register(7), dst, 0);
5652 }
5653 else {
5654 emit_int8((unsigned char)0xC1);
5655 emit_operand(as_Register(7), dst, 1);
5656 emit_int8(imm8);
5657 }
5658 }
5659
5660 void Assembler::sarl(Address dst) {
5661 InstructionMark im(this);
5662 prefix(dst);
5663 emit_int8((unsigned char)0xD3);
5664 emit_operand(as_Register(7), dst, 0);
5665 }
5666
5667 void Assembler::sarl(Register dst, int imm8) {
5668 int encode = prefix_and_encode(dst->encoding());
5669 assert(isShiftCount(imm8), "illegal shift count");
5670 if (imm8 == 1) {
5671 emit_int16((unsigned char)0xD1, (0xF8 | encode));
5672 } else {
5673 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
5674 }
5675 }
5676
5677 void Assembler::sarl(Register dst) {
5678 int encode = prefix_and_encode(dst->encoding());
5679 emit_int16((unsigned char)0xD3, (0xF8 | encode));
5680 }
5681
5682 void Assembler::sbbl(Address dst, int32_t imm32) {
5683 InstructionMark im(this);
5684 prefix(dst);
5685 emit_arith_operand(0x81, rbx, dst, imm32);
5686 }
5687
5688 void Assembler::sbbl(Register dst, int32_t imm32) {
5689 prefix(dst);
5690 emit_arith(0x81, 0xD8, dst, imm32);
5691 }
5692
5693
5694 void Assembler::sbbl(Register dst, Address src) {
5695 InstructionMark im(this);
5696 prefix(src, dst);
5697 emit_int8(0x1B);
5698 emit_operand(dst, src, 0);
5699 }
5700
5701 void Assembler::sbbl(Register dst, Register src) {
5702 (void) prefix_and_encode(dst->encoding(), src->encoding());
5703 emit_arith(0x1B, 0xC0, dst, src);
5704 }
5705
5706 void Assembler::setb(Condition cc, Register dst) {
5707 assert(0 <= cc && cc < 16, "illegal cc");
5708 int encode = prefix_and_encode(dst->encoding(), true);
5709 emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
5710 }
5711
5712 void Assembler::sete(Register dst) {
5713 int encode = prefix_and_encode(dst->encoding(), true);
5714 emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
5715 }
5716
5717 void Assembler::setl(Register dst) {
5718 int encode = prefix_and_encode(dst->encoding(), true);
5719 emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
5720 }
5721
5722 void Assembler::setne(Register dst) {
5723 int encode = prefix_and_encode(dst->encoding(), true);
5724 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
5725 }
5726
5727 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
5728 assert(VM_Version::supports_ssse3(), "");
5729 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5730 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5731 emit_int24(0x0F, (0xC0 | encode), imm8);
5732 }
5733
5734 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5735 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5736 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5737 0, "");
5738 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5739 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5740 emit_int24(0x0F, (0xC0 | encode), imm8);
5741 }
5742
5743 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5744 assert(VM_Version::supports_evex(), "");
5745 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5746 attributes.set_is_evex_instruction();
5747 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5748 emit_int24(0x3, (0xC0 | encode), imm8);
5749 }
5750
5751 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
5752 assert(VM_Version::supports_sse4_1(), "");
5753 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5754 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5755 emit_int24(0x0E, (0xC0 | encode), imm8);
5756 }
5757
5758 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
5759 assert(VM_Version::supports_sha(), "");
5760 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
5761 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
5762 }
5763
5764 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
5765 assert(VM_Version::supports_sha(), "");
5766 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5767 emit_int16((unsigned char)0xC8, (0xC0 | encode));
5768 }
5769
5770 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
5771 assert(VM_Version::supports_sha(), "");
5772 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5773 emit_int16((unsigned char)0xC9, (0xC0 | encode));
5774 }
5775
5776 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
5777 assert(VM_Version::supports_sha(), "");
5778 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5779 emit_int16((unsigned char)0xCA, (0xC0 | encode));
5780 }
5781
5782 // xmm0 is implicit additional source to this instruction.
5783 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
5784 assert(VM_Version::supports_sha(), "");
5785 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5786 emit_int16((unsigned char)0xCB, (0xC0 | encode));
5787 }
5788
5789 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
5790 assert(VM_Version::supports_sha(), "");
5791 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5792 emit_int16((unsigned char)0xCC, (0xC0 | encode));
5793 }
5794
5795 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
5796 assert(VM_Version::supports_sha(), "");
5797 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5798 emit_int16((unsigned char)0xCD, (0xC0 | encode));
5799 }
5800
5801
5802 void Assembler::shll(Register dst, int imm8) {
5803 assert(isShiftCount(imm8), "illegal shift count");
5804 int encode = prefix_and_encode(dst->encoding());
5805 if (imm8 == 1 ) {
5806 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5807 } else {
5808 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5809 }
5810 }
5811
5812 void Assembler::shll(Register dst) {
5813 int encode = prefix_and_encode(dst->encoding());
5814 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5815 }
5816
5817 void Assembler::shrl(Register dst, int imm8) {
5818 assert(isShiftCount(imm8), "illegal shift count");
5819 int encode = prefix_and_encode(dst->encoding());
5820 if (imm8 == 1) {
5821 emit_int16((unsigned char)0xD1, (0xE8 | encode));
5822 }
5823 else {
5824 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
5825 }
5826 }
5827
5828 void Assembler::shrl(Register dst) {
5829 int encode = prefix_and_encode(dst->encoding());
5830 emit_int16((unsigned char)0xD3, (0xE8 | encode));
5831 }
5832
5833 void Assembler::shrl(Address dst) {
5834 InstructionMark im(this);
5835 prefix(dst);
5836 emit_int8((unsigned char)0xD3);
5837 emit_operand(as_Register(5), dst, 0);
5838 }
5839
5840 void Assembler::shrl(Address dst, int imm8) {
5841 InstructionMark im(this);
5842 assert(isShiftCount(imm8), "illegal shift count");
5843 prefix(dst);
5844 if (imm8 == 1) {
5845 emit_int8((unsigned char)0xD1);
5846 emit_operand(as_Register(5), dst, 0);
5847 }
5848 else {
5849 emit_int8((unsigned char)0xC1);
5850 emit_operand(as_Register(5), dst, 1);
5851 emit_int8(imm8);
5852 }
5853 }
5854
5855
5856 void Assembler::shldl(Register dst, Register src) {
5857 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5858 emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
5859 }
5860
5861 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
5862 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5863 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
5864 }
5865
5866 void Assembler::shrdl(Register dst, Register src) {
5867 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5868 emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
5869 }
5870
5871 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
5872 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5873 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
5874 }
5875
5876 // copies a single word from [esi] to [edi]
5877 void Assembler::smovl() {
5878 emit_int8((unsigned char)0xA5);
5879 }
5880
5881 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
5882 assert(VM_Version::supports_sse4_1(), "");
5883 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5884 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5885 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
5886 }
5887
5888 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
5889 assert(VM_Version::supports_sse4_1(), "");
5890 InstructionMark im(this);
5891 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5892 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5893 emit_int8(0x0B);
5894 emit_operand(dst, src, 1);
5895 emit_int8((unsigned char)rmode);
5896 }
5897
5898 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
5899 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5900 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5901 attributes.set_rex_vex_w_reverted();
5902 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5903 emit_int16(0x51, (0xC0 | encode));
5904 }
5905
5906 void Assembler::sqrtsd(XMMRegister dst, Address src) {
5907 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5908 InstructionMark im(this);
5909 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5910 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5911 attributes.set_rex_vex_w_reverted();
5912 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5913 emit_int8(0x51);
5914 emit_operand(dst, src, 0);
5915 }
5916
5917 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
5918 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5919 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5920 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5921 emit_int16(0x51, (0xC0 | encode));
5922 }
5923
5924 void Assembler::std() {
5925 emit_int8((unsigned char)0xFD);
5926 }
5927
5928 void Assembler::sqrtss(XMMRegister dst, Address src) {
5929 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5930 InstructionMark im(this);
5931 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5932 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5933 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5934 emit_int8(0x51);
5935 emit_operand(dst, src, 0);
5936 }
5937
5938 void Assembler::stmxcsr( Address dst) {
5939 if (UseAVX > 0 ) {
5940 assert(VM_Version::supports_avx(), "");
5941 InstructionMark im(this);
5942 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5943 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5944 emit_int8((unsigned char)0xAE);
5945 emit_operand(as_Register(3), dst, 0);
5946 } else {
5947 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5948 InstructionMark im(this);
5949 prefix(dst);
5950 emit_int16(0x0F, (unsigned char)0xAE);
5951 emit_operand(as_Register(3), dst, 0);
5952 }
5953 }
5954
5955 void Assembler::subl(Address dst, int32_t imm32) {
5956 InstructionMark im(this);
5957 prefix(dst);
5958 emit_arith_operand(0x81, rbp, dst, imm32);
5959 }
5960
5961 void Assembler::subl(Address dst, Register src) {
5962 InstructionMark im(this);
5963 prefix(dst, src);
5964 emit_int8(0x29);
5965 emit_operand(src, dst, 0);
5966 }
5967
5968 void Assembler::subl(Register dst, int32_t imm32) {
5969 prefix(dst);
5970 emit_arith(0x81, 0xE8, dst, imm32);
5971 }
5972
5973 // Force generation of a 4 byte immediate value even if it fits into 8bit
5974 void Assembler::subl_imm32(Register dst, int32_t imm32) {
5975 prefix(dst);
5976 emit_arith_imm32(0x81, 0xE8, dst, imm32);
5977 }
5978
5979 void Assembler::subl(Register dst, Address src) {
5980 InstructionMark im(this);
5981 prefix(src, dst);
5982 emit_int8(0x2B);
5983 emit_operand(dst, src, 0);
5984 }
5985
5986 void Assembler::subl(Register dst, Register src) {
5987 (void) prefix_and_encode(dst->encoding(), src->encoding());
5988 emit_arith(0x2B, 0xC0, dst, src);
5989 }
5990
5991 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
5992 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5993 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5994 attributes.set_rex_vex_w_reverted();
5995 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5996 emit_int16(0x5C, (0xC0 | encode));
5997 }
5998
5999 void Assembler::subsd(XMMRegister dst, Address src) {
6000 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6001 InstructionMark im(this);
6002 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6003 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6004 attributes.set_rex_vex_w_reverted();
6005 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6006 emit_int8(0x5C);
6007 emit_operand(dst, src, 0);
6008 }
6009
6010 void Assembler::subss(XMMRegister dst, XMMRegister src) {
6011 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6012 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
6013 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6014 emit_int16(0x5C, (0xC0 | encode));
6015 }
6016
6017 void Assembler::subss(XMMRegister dst, Address src) {
6018 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6019 InstructionMark im(this);
6020 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6021 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6022 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6023 emit_int8(0x5C);
6024 emit_operand(dst, src, 0);
6025 }
6026
6027 void Assembler::testb(Register dst, int imm8) {
6028 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
6029 if (dst == rax) {
6030 emit_int8((unsigned char)0xA8);
6031 emit_int8(imm8);
6032 } else {
6033 (void) prefix_and_encode(dst->encoding(), true);
6034 emit_arith_b(0xF6, 0xC0, dst, imm8);
6035 }
6036 }
6037
6038 void Assembler::testb(Address dst, int imm8) {
6039 InstructionMark im(this);
6040 prefix(dst);
6041 emit_int8((unsigned char)0xF6);
6042 emit_operand(rax, dst, 1);
6043 emit_int8(imm8);
6044 }
6045
6046 void Assembler::testl(Address dst, int32_t imm32) {
6047 InstructionMark im(this);
6048 prefix(dst);
6049 emit_int8((unsigned char)0xF7);
6050 emit_operand(as_Register(0), dst, 4);
6051 emit_int32(imm32);
6052 }
6053
6054 void Assembler::testl(Register dst, int32_t imm32) {
6055 // not using emit_arith because test
6056 // doesn't support sign-extension of
6057 // 8bit operands
6058 if (dst == rax) {
6059 emit_int8((unsigned char)0xA9);
6060 emit_int32(imm32);
6061 } else {
6062 int encode = dst->encoding();
6063 encode = prefix_and_encode(encode);
6064 emit_int16((unsigned char)0xF7, (0xC0 | encode));
6065 emit_int32(imm32);
6066 }
6067 }
6068
6069 void Assembler::testl(Register dst, Register src) {
6070 (void) prefix_and_encode(dst->encoding(), src->encoding());
6071 emit_arith(0x85, 0xC0, dst, src);
6072 }
6073
6074 void Assembler::testl(Register dst, Address src) {
6075 InstructionMark im(this);
6076 prefix(src, dst);
6077 emit_int8((unsigned char)0x85);
6078 emit_operand(dst, src, 0);
6079 }
6080
6081 void Assembler::tzcntl(Register dst, Register src) {
6082 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6083 emit_int8((unsigned char)0xF3);
6084 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6085 emit_int24(0x0F,
6086 (unsigned char)0xBC,
6087 0xC0 | encode);
6088 }
6089
6090 void Assembler::tzcntl(Register dst, Address src) {
6091 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6092 InstructionMark im(this);
6093 emit_int8((unsigned char)0xF3);
6094 prefix(src, dst);
6095 emit_int16(0x0F, (unsigned char)0xBC);
6096 emit_operand(dst, src, 0);
6097 }
6098
6099 void Assembler::tzcntq(Register dst, Register src) {
6100 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6101 emit_int8((unsigned char)0xF3);
6102 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6103 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
6104 }
6105
6106 void Assembler::tzcntq(Register dst, Address src) {
6107 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6108 InstructionMark im(this);
6109 emit_int8((unsigned char)0xF3);
6110 prefixq(src, dst);
6111 emit_int16(0x0F, (unsigned char)0xBC);
6112 emit_operand(dst, src, 0);
6113 }
6114
6115 void Assembler::ucomisd(XMMRegister dst, Address src) {
6116 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6117 InstructionMark im(this);
6118 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6119 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6120 attributes.set_rex_vex_w_reverted();
6121 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6122 emit_int8(0x2E);
6123 emit_operand(dst, src, 0);
6124 }
6125
6126 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
6127 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6128 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6129 attributes.set_rex_vex_w_reverted();
6130 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6131 emit_int16(0x2E, (0xC0 | encode));
6132 }
6133
6134 void Assembler::ucomiss(XMMRegister dst, Address src) {
6135 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6136 InstructionMark im(this);
6137 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6138 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6139 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6140 emit_int8(0x2E);
6141 emit_operand(dst, src, 0);
6142 }
6143
6144 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
6145 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6146 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6147 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6148 emit_int16(0x2E, (0xC0 | encode));
6149 }
6150
6151 void Assembler::xabort(int8_t imm8) {
6152 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
6153 }
6154
6155 void Assembler::xaddb(Address dst, Register src) {
6156 InstructionMark im(this);
6157 prefix(dst, src, true);
6158 emit_int16(0x0F, (unsigned char)0xC0);
6159 emit_operand(src, dst, 0);
6160 }
6161
6162 void Assembler::xaddw(Address dst, Register src) {
6163 InstructionMark im(this);
6164 emit_int8(0x66);
6165 prefix(dst, src);
6166 emit_int16(0x0F, (unsigned char)0xC1);
6167 emit_operand(src, dst, 0);
6168 }
6169
6170 void Assembler::xaddl(Address dst, Register src) {
6171 InstructionMark im(this);
6172 prefix(dst, src);
6173 emit_int16(0x0F, (unsigned char)0xC1);
6174 emit_operand(src, dst, 0);
6175 }
6176
6177 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
6178 InstructionMark im(this);
6179 relocate(rtype);
6180 if (abort.is_bound()) {
6181 address entry = target(abort);
6182 assert(entry != NULL, "abort entry NULL");
6183 intptr_t offset = entry - pc();
6184 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6185 emit_int32(offset - 6); // 2 opcode + 4 address
6186 } else {
6187 abort.add_patch_at(code(), locator());
6188 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6189 emit_int32(0);
6190 }
6191 }
6192
6193 void Assembler::xchgb(Register dst, Address src) { // xchg
6194 InstructionMark im(this);
6195 prefix(src, dst, true);
6196 emit_int8((unsigned char)0x86);
6197 emit_operand(dst, src, 0);
6198 }
6199
6200 void Assembler::xchgw(Register dst, Address src) { // xchg
6201 InstructionMark im(this);
6202 emit_int8(0x66);
6203 prefix(src, dst);
6204 emit_int8((unsigned char)0x87);
6205 emit_operand(dst, src, 0);
6206 }
6207
6208 void Assembler::xchgl(Register dst, Address src) { // xchg
6209 InstructionMark im(this);
6210 prefix(src, dst);
6211 emit_int8((unsigned char)0x87);
6212 emit_operand(dst, src, 0);
6213 }
6214
6215 void Assembler::xchgl(Register dst, Register src) {
6216 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6217 emit_int16((unsigned char)0x87, (0xC0 | encode));
6218 }
6219
6220 void Assembler::xend() {
6221 emit_int24(0x0F, 0x01, (unsigned char)0xD5);
6222 }
6223
6224 void Assembler::xgetbv() {
6225 emit_int24(0x0F, 0x01, (unsigned char)0xD0);
6226 }
6227
6228 void Assembler::xorl(Address dst, int32_t imm32) {
6229 InstructionMark im(this);
6230 prefix(dst);
6231 emit_arith_operand(0x81, as_Register(6), dst, imm32);
6232 }
6233
6234 void Assembler::xorl(Register dst, int32_t imm32) {
6235 prefix(dst);
6236 emit_arith(0x81, 0xF0, dst, imm32);
6237 }
6238
6239 void Assembler::xorl(Register dst, Address src) {
6240 InstructionMark im(this);
6241 prefix(src, dst);
6242 emit_int8(0x33);
6243 emit_operand(dst, src, 0);
6244 }
6245
6246 void Assembler::xorl(Register dst, Register src) {
6247 (void) prefix_and_encode(dst->encoding(), src->encoding());
6248 emit_arith(0x33, 0xC0, dst, src);
6249 }
6250
6251 void Assembler::xorl(Address dst, Register src) {
6252 InstructionMark im(this);
6253 prefix(dst, src);
6254 emit_int8(0x31);
6255 emit_operand(src, dst, 0);
6256 }
6257
6258 void Assembler::xorb(Register dst, Address src) {
6259 InstructionMark im(this);
6260 prefix(src, dst);
6261 emit_int8(0x32);
6262 emit_operand(dst, src, 0);
6263 }
6264
6265 void Assembler::xorb(Address dst, Register src) {
6266 InstructionMark im(this);
6267 prefix(dst, src, true);
6268 emit_int8(0x30);
6269 emit_operand(src, dst, 0);
6270 }
6271
6272 void Assembler::xorw(Register dst, Register src) {
6273 (void)prefix_and_encode(dst->encoding(), src->encoding());
6274 emit_arith(0x33, 0xC0, dst, src);
6275 }
6276
6277 // AVX 3-operands scalar float-point arithmetic instructions
6278
6279 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
6280 assert(VM_Version::supports_avx(), "");
6281 InstructionMark im(this);
6282 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6283 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6284 attributes.set_rex_vex_w_reverted();
6285 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6286 emit_int8(0x58);
6287 emit_operand(dst, src, 0);
6288 }
6289
6290 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6291 assert(VM_Version::supports_avx(), "");
6292 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6293 attributes.set_rex_vex_w_reverted();
6294 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6295 emit_int16(0x58, (0xC0 | encode));
6296 }
6297
6298 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
6299 assert(VM_Version::supports_avx(), "");
6300 InstructionMark im(this);
6301 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6302 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6303 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6304 emit_int8(0x58);
6305 emit_operand(dst, src, 0);
6306 }
6307
6308 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6309 assert(VM_Version::supports_avx(), "");
6310 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6311 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6312 emit_int16(0x58, (0xC0 | encode));
6313 }
6314
6315 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
6316 assert(VM_Version::supports_avx(), "");
6317 InstructionMark im(this);
6318 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6319 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6320 attributes.set_rex_vex_w_reverted();
6321 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6322 emit_int8(0x5E);
6323 emit_operand(dst, src, 0);
6324 }
6325
6326 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6327 assert(VM_Version::supports_avx(), "");
6328 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6329 attributes.set_rex_vex_w_reverted();
6330 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6331 emit_int16(0x5E, (0xC0 | encode));
6332 }
6333
6334 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
6335 assert(VM_Version::supports_avx(), "");
6336 InstructionMark im(this);
6337 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6338 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6339 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6340 emit_int8(0x5E);
6341 emit_operand(dst, src, 0);
6342 }
6343
6344 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6345 assert(VM_Version::supports_avx(), "");
6346 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6347 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6348 emit_int16(0x5E, (0xC0 | encode));
6349 }
6350
6351 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6352 assert(VM_Version::supports_fma(), "");
6353 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6354 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6355 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6356 }
6357
6358 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6359 assert(VM_Version::supports_fma(), "");
6360 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6361 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6362 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6363 }
6364
6365 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
6366 assert(VM_Version::supports_avx(), "");
6367 InstructionMark im(this);
6368 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6369 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6370 attributes.set_rex_vex_w_reverted();
6371 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6372 emit_int8(0x59);
6373 emit_operand(dst, src, 0);
6374 }
6375
6376 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6377 assert(VM_Version::supports_avx(), "");
6378 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6379 attributes.set_rex_vex_w_reverted();
6380 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6381 emit_int16(0x59, (0xC0 | encode));
6382 }
6383
6384 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
6385 assert(VM_Version::supports_avx(), "");
6386 InstructionMark im(this);
6387 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6388 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6389 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6390 emit_int8(0x59);
6391 emit_operand(dst, src, 0);
6392 }
6393
6394 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6395 assert(VM_Version::supports_avx(), "");
6396 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6397 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6398 emit_int16(0x59, (0xC0 | encode));
6399 }
6400
6401 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
6402 assert(VM_Version::supports_avx(), "");
6403 InstructionMark im(this);
6404 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6405 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6406 attributes.set_rex_vex_w_reverted();
6407 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6408 emit_int8(0x5C);
6409 emit_operand(dst, src, 0);
6410 }
6411
6412 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6413 assert(VM_Version::supports_avx(), "");
6414 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6415 attributes.set_rex_vex_w_reverted();
6416 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6417 emit_int16(0x5C, (0xC0 | encode));
6418 }
6419
6420 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
6421 assert(VM_Version::supports_avx(), "");
6422 InstructionMark im(this);
6423 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6424 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6425 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6426 emit_int8(0x5C);
6427 emit_operand(dst, src, 0);
6428 }
6429
6430 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6431 assert(VM_Version::supports_avx(), "");
6432 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6433 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6434 emit_int16(0x5C, (0xC0 | encode));
6435 }
6436
6437 //====================VECTOR ARITHMETIC=====================================
6438
6439 // Float-point vector arithmetic
6440
6441 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
6442 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6443 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6444 attributes.set_rex_vex_w_reverted();
6445 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6446 emit_int16(0x58, (0xC0 | encode));
6447 }
6448
6449 void Assembler::addpd(XMMRegister dst, Address src) {
6450 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6451 InstructionMark im(this);
6452 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6453 attributes.set_rex_vex_w_reverted();
6454 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6455 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6456 emit_int8(0x58);
6457 emit_operand(dst, src, 0);
6458 }
6459
6460
6461 void Assembler::addps(XMMRegister dst, XMMRegister src) {
6462 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6463 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6464 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6465 emit_int16(0x58, (0xC0 | encode));
6466 }
6467
6468 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6469 assert(VM_Version::supports_avx(), "");
6470 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6471 attributes.set_rex_vex_w_reverted();
6472 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6473 emit_int16(0x58, (0xC0 | encode));
6474 }
6475
6476 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6477 assert(VM_Version::supports_avx(), "");
6478 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6479 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6480 emit_int16(0x58, (0xC0 | encode));
6481 }
6482
6483 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6484 assert(VM_Version::supports_avx(), "");
6485 InstructionMark im(this);
6486 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6487 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6488 attributes.set_rex_vex_w_reverted();
6489 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6490 emit_int8(0x58);
6491 emit_operand(dst, src, 0);
6492 }
6493
6494 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6495 assert(VM_Version::supports_avx(), "");
6496 InstructionMark im(this);
6497 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6498 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6499 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6500 emit_int8(0x58);
6501 emit_operand(dst, src, 0);
6502 }
6503
6504 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
6505 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6506 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6507 attributes.set_rex_vex_w_reverted();
6508 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6509 emit_int16(0x5C, (0xC0 | encode));
6510 }
6511
6512 void Assembler::subps(XMMRegister dst, XMMRegister src) {
6513 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6514 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6515 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6516 emit_int16(0x5C, (0xC0 | encode));
6517 }
6518
6519 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6520 assert(VM_Version::supports_avx(), "");
6521 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6522 attributes.set_rex_vex_w_reverted();
6523 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6524 emit_int16(0x5C, (0xC0 | encode));
6525 }
6526
6527 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6528 assert(VM_Version::supports_avx(), "");
6529 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6530 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6531 emit_int16(0x5C, (0xC0 | encode));
6532 }
6533
6534 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6535 assert(VM_Version::supports_avx(), "");
6536 InstructionMark im(this);
6537 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6538 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6539 attributes.set_rex_vex_w_reverted();
6540 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6541 emit_int8(0x5C);
6542 emit_operand(dst, src, 0);
6543 }
6544
6545 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6546 assert(VM_Version::supports_avx(), "");
6547 InstructionMark im(this);
6548 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6549 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6550 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6551 emit_int8(0x5C);
6552 emit_operand(dst, src, 0);
6553 }
6554
6555 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
6556 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6557 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6558 attributes.set_rex_vex_w_reverted();
6559 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6560 emit_int16(0x59, (0xC0 | encode));
6561 }
6562
6563 void Assembler::mulpd(XMMRegister dst, Address src) {
6564 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6565 InstructionMark im(this);
6566 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6567 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6568 attributes.set_rex_vex_w_reverted();
6569 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6570 emit_int8(0x59);
6571 emit_operand(dst, src, 0);
6572 }
6573
6574 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
6575 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6576 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6577 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6578 emit_int16(0x59, (0xC0 | encode));
6579 }
6580
6581 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6582 assert(VM_Version::supports_avx(), "");
6583 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6584 attributes.set_rex_vex_w_reverted();
6585 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6586 emit_int16(0x59, (0xC0 | encode));
6587 }
6588
6589 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6590 assert(VM_Version::supports_avx(), "");
6591 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6592 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6593 emit_int16(0x59, (0xC0 | encode));
6594 }
6595
6596 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6597 assert(VM_Version::supports_avx(), "");
6598 InstructionMark im(this);
6599 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6600 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6601 attributes.set_rex_vex_w_reverted();
6602 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6603 emit_int8(0x59);
6604 emit_operand(dst, src, 0);
6605 }
6606
6607 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6608 assert(VM_Version::supports_avx(), "");
6609 InstructionMark im(this);
6610 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6611 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6612 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6613 emit_int8(0x59);
6614 emit_operand(dst, src, 0);
6615 }
6616
6617 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6618 assert(VM_Version::supports_fma(), "");
6619 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6620 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6621 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6622 }
6623
6624 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6625 assert(VM_Version::supports_fma(), "");
6626 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6627 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6628 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6629 }
6630
6631 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6632 assert(VM_Version::supports_fma(), "");
6633 InstructionMark im(this);
6634 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6635 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6636 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6637 emit_int8((unsigned char)0xB8);
6638 emit_operand(dst, src2, 0);
6639 }
6640
6641 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6642 assert(VM_Version::supports_fma(), "");
6643 InstructionMark im(this);
6644 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6645 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6646 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6647 emit_int8((unsigned char)0xB8);
6648 emit_operand(dst, src2, 0);
6649 }
6650
6651 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
6652 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6653 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6654 attributes.set_rex_vex_w_reverted();
6655 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6656 emit_int16(0x5E, (0xC0 | encode));
6657 }
6658
6659 void Assembler::divps(XMMRegister dst, XMMRegister src) {
6660 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6661 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6663 emit_int16(0x5E, (0xC0 | encode));
6664 }
6665
6666 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6667 assert(VM_Version::supports_avx(), "");
6668 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6669 attributes.set_rex_vex_w_reverted();
6670 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6671 emit_int16(0x5E, (0xC0 | encode));
6672 }
6673
6674 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6675 assert(VM_Version::supports_avx(), "");
6676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6677 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6678 emit_int16(0x5E, (0xC0 | encode));
6679 }
6680
6681 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6682 assert(VM_Version::supports_avx(), "");
6683 InstructionMark im(this);
6684 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6685 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6686 attributes.set_rex_vex_w_reverted();
6687 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6688 emit_int8(0x5E);
6689 emit_operand(dst, src, 0);
6690 }
6691
6692 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6693 assert(VM_Version::supports_avx(), "");
6694 InstructionMark im(this);
6695 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6696 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6697 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6698 emit_int8(0x5E);
6699 emit_operand(dst, src, 0);
6700 }
6701
6702 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6703 assert(VM_Version::supports_avx(), "");
6704 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6705 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6706 emit_int24(0x09, (0xC0 | encode), (rmode));
6707 }
6708
6709 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6710 assert(VM_Version::supports_avx(), "");
6711 InstructionMark im(this);
6712 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6713 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6714 emit_int8(0x09);
6715 emit_operand(dst, src, 1);
6716 emit_int8((rmode));
6717 }
6718
6719 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6720 assert(VM_Version::supports_evex(), "requires EVEX support");
6721 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6722 attributes.set_is_evex_instruction();
6723 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6724 emit_int24(0x09, (0xC0 | encode), (rmode));
6725 }
6726
6727 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6728 assert(VM_Version::supports_evex(), "requires EVEX support");
6729 assert(dst != xnoreg, "sanity");
6730 InstructionMark im(this);
6731 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6732 attributes.set_is_evex_instruction();
6733 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6734 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6735 emit_int8(0x09);
6736 emit_operand(dst, src, 1);
6737 emit_int8((rmode));
6738 }
6739
6740 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
6741 assert(VM_Version::supports_avx(), "");
6742 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6743 attributes.set_rex_vex_w_reverted();
6744 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6745 emit_int16(0x51, (0xC0 | encode));
6746 }
6747
6748 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
6749 assert(VM_Version::supports_avx(), "");
6750 InstructionMark im(this);
6751 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6752 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6753 attributes.set_rex_vex_w_reverted();
6754 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6755 emit_int8(0x51);
6756 emit_operand(dst, src, 0);
6757 }
6758
6759 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
6760 assert(VM_Version::supports_avx(), "");
6761 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6762 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6763 emit_int16(0x51, (0xC0 | encode));
6764 }
6765
6766 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
6767 assert(VM_Version::supports_avx(), "");
6768 InstructionMark im(this);
6769 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6770 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6771 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6772 emit_int8(0x51);
6773 emit_operand(dst, src, 0);
6774 }
6775
6776 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
6777 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6778 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6779 attributes.set_rex_vex_w_reverted();
6780 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6781 emit_int16(0x54, (0xC0 | encode));
6782 }
6783
6784 void Assembler::andps(XMMRegister dst, XMMRegister src) {
6785 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6786 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6787 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6788 emit_int16(0x54, (0xC0 | encode));
6789 }
6790
6791 void Assembler::andps(XMMRegister dst, Address src) {
6792 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6793 InstructionMark im(this);
6794 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6795 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6796 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6797 emit_int8(0x54);
6798 emit_operand(dst, src, 0);
6799 }
6800
6801 void Assembler::andpd(XMMRegister dst, Address src) {
6802 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6803 InstructionMark im(this);
6804 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6805 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6806 attributes.set_rex_vex_w_reverted();
6807 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6808 emit_int8(0x54);
6809 emit_operand(dst, src, 0);
6810 }
6811
6812 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6813 assert(VM_Version::supports_avx(), "");
6814 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6815 attributes.set_rex_vex_w_reverted();
6816 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6817 emit_int16(0x54, (0xC0 | encode));
6818 }
6819
6820 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6821 assert(VM_Version::supports_avx(), "");
6822 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6823 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6824 emit_int16(0x54, (0xC0 | encode));
6825 }
6826
6827 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6828 assert(VM_Version::supports_avx(), "");
6829 InstructionMark im(this);
6830 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6831 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6832 attributes.set_rex_vex_w_reverted();
6833 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6834 emit_int8(0x54);
6835 emit_operand(dst, src, 0);
6836 }
6837
6838 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6839 assert(VM_Version::supports_avx(), "");
6840 InstructionMark im(this);
6841 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6842 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6843 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6844 emit_int8(0x54);
6845 emit_operand(dst, src, 0);
6846 }
6847
6848 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
6849 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6850 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6851 attributes.set_rex_vex_w_reverted();
6852 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6853 emit_int8(0x15);
6854 emit_int8((0xC0 | encode));
6855 }
6856
6857 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
6858 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6859 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6860 attributes.set_rex_vex_w_reverted();
6861 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6862 emit_int16(0x14, (0xC0 | encode));
6863 }
6864
6865 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
6866 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6867 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6868 attributes.set_rex_vex_w_reverted();
6869 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6870 emit_int16(0x57, (0xC0 | encode));
6871 }
6872
6873 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
6874 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6875 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6876 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6877 emit_int16(0x57, (0xC0 | encode));
6878 }
6879
6880 void Assembler::xorpd(XMMRegister dst, Address src) {
6881 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6882 InstructionMark im(this);
6883 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6884 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6885 attributes.set_rex_vex_w_reverted();
6886 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6887 emit_int8(0x57);
6888 emit_operand(dst, src, 0);
6889 }
6890
6891 void Assembler::xorps(XMMRegister dst, Address src) {
6892 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6893 InstructionMark im(this);
6894 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6895 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6896 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6897 emit_int8(0x57);
6898 emit_operand(dst, src, 0);
6899 }
6900
6901 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6902 assert(VM_Version::supports_avx(), "");
6903 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6904 attributes.set_rex_vex_w_reverted();
6905 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6906 emit_int16(0x57, (0xC0 | encode));
6907 }
6908
6909 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6910 assert(VM_Version::supports_avx(), "");
6911 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6912 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6913 emit_int16(0x57, (0xC0 | encode));
6914 }
6915
6916 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6917 assert(VM_Version::supports_avx(), "");
6918 InstructionMark im(this);
6919 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6920 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6921 attributes.set_rex_vex_w_reverted();
6922 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6923 emit_int8(0x57);
6924 emit_operand(dst, src, 0);
6925 }
6926
6927 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6928 assert(VM_Version::supports_avx(), "");
6929 InstructionMark im(this);
6930 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6931 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6932 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6933 emit_int8(0x57);
6934 emit_operand(dst, src, 0);
6935 }
6936
6937 // Integer vector arithmetic
6938 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6939 assert(VM_Version::supports_avx() && (vector_len == 0) ||
6940 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
6941 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6942 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6943 emit_int16(0x01, (0xC0 | encode));
6944 }
6945
6946 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6947 assert(VM_Version::supports_avx() && (vector_len == 0) ||
6948 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
6949 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6950 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6951 emit_int16(0x02, (0xC0 | encode));
6952 }
6953
6954 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
6955 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6956 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6957 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6958 emit_int16((unsigned char)0xFC, (0xC0 | encode));
6959 }
6960
6961 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
6962 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6964 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6965 emit_int16((unsigned char)0xFD, (0xC0 | encode));
6966 }
6967
6968 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
6969 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6970 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6971 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6972 emit_int16((unsigned char)0xFE, (0xC0 | encode));
6973 }
6974
6975 void Assembler::paddd(XMMRegister dst, Address src) {
6976 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6977 InstructionMark im(this);
6978 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6979 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6980 emit_int8((unsigned char)0xFE);
6981 emit_operand(dst, src, 0);
6982 }
6983
6984 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
6985 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6986 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6987 attributes.set_rex_vex_w_reverted();
6988 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6989 emit_int16((unsigned char)0xD4, (0xC0 | encode));
6990 }
6991
6992 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
6993 assert(VM_Version::supports_sse3(), "");
6994 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6995 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6996 emit_int16(0x01, (0xC0 | encode));
6997 }
6998
6999 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
7000 assert(VM_Version::supports_sse3(), "");
7001 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7002 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7003 emit_int16(0x02, (0xC0 | encode));
7004 }
7005
7006 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7007 assert(UseAVX > 0, "requires some form of AVX");
7008 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7009 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7010 emit_int16((unsigned char)0xFC, (0xC0 | encode));
7011 }
7012
7013 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7014 assert(UseAVX > 0, "requires some form of AVX");
7015 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7016 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7017 emit_int16((unsigned char)0xFD, (0xC0 | encode));
7018 }
7019
7020 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7021 assert(UseAVX > 0, "requires some form of AVX");
7022 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7023 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7024 emit_int16((unsigned char)0xFE, (0xC0 | encode));
7025 }
7026
7027 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7028 assert(UseAVX > 0, "requires some form of AVX");
7029 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7030 attributes.set_rex_vex_w_reverted();
7031 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7032 emit_int16((unsigned char)0xD4, (0xC0 | encode));
7033 }
7034
7035 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7036 assert(UseAVX > 0, "requires some form of AVX");
7037 InstructionMark im(this);
7038 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7039 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7040 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7041 emit_int8((unsigned char)0xFC);
7042 emit_operand(dst, src, 0);
7043 }
7044
7045 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7046 assert(UseAVX > 0, "requires some form of AVX");
7047 InstructionMark im(this);
7048 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7049 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7050 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7051 emit_int8((unsigned char)0xFD);
7052 emit_operand(dst, src, 0);
7053 }
7054
7055 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7056 assert(UseAVX > 0, "requires some form of AVX");
7057 InstructionMark im(this);
7058 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7059 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7060 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7061 emit_int8((unsigned char)0xFE);
7062 emit_operand(dst, src, 0);
7063 }
7064
7065 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7066 assert(UseAVX > 0, "requires some form of AVX");
7067 InstructionMark im(this);
7068 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7069 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7070 attributes.set_rex_vex_w_reverted();
7071 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7072 emit_int8((unsigned char)0xD4);
7073 emit_operand(dst, src, 0);
7074 }
7075
7076 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
7077 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7078 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7079 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7080 emit_int16((unsigned char)0xF8, (0xC0 | encode));
7081 }
7082
7083 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
7084 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7085 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7086 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7087 emit_int16((unsigned char)0xF9, (0xC0 | encode));
7088 }
7089
7090 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
7091 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7092 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7093 emit_int16((unsigned char)0xFA, (0xC0 | encode));
7094 }
7095
7096 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
7097 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7098 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7099 attributes.set_rex_vex_w_reverted();
7100 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7101 emit_int8((unsigned char)0xFB);
7102 emit_int8((0xC0 | encode));
7103 }
7104
7105 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7106 assert(UseAVX > 0, "requires some form of AVX");
7107 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7108 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7109 emit_int16((unsigned char)0xD8, (0xC0 | encode));
7110 }
7111
7112 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7113 assert(UseAVX > 0, "requires some form of AVX");
7114 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7115 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7116 emit_int16((unsigned char)0xF8, (0xC0 | encode));
7117 }
7118
7119 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7120 assert(UseAVX > 0, "requires some form of AVX");
7121 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7122 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7123 emit_int16((unsigned char)0xF9, (0xC0 | encode));
7124 }
7125
7126 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7127 assert(UseAVX > 0, "requires some form of AVX");
7128 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7129 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7130 emit_int16((unsigned char)0xFA, (0xC0 | encode));
7131 }
7132
7133 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7134 assert(UseAVX > 0, "requires some form of AVX");
7135 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7136 attributes.set_rex_vex_w_reverted();
7137 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7138 emit_int16((unsigned char)0xFB, (0xC0 | encode));
7139 }
7140
7141 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7142 assert(UseAVX > 0, "requires some form of AVX");
7143 InstructionMark im(this);
7144 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7145 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7146 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7147 emit_int8((unsigned char)0xF8);
7148 emit_operand(dst, src, 0);
7149 }
7150
7151 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7152 assert(UseAVX > 0, "requires some form of AVX");
7153 InstructionMark im(this);
7154 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7155 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7156 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7157 emit_int8((unsigned char)0xF9);
7158 emit_operand(dst, src, 0);
7159 }
7160
7161 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7162 assert(UseAVX > 0, "requires some form of AVX");
7163 InstructionMark im(this);
7164 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7165 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7166 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7167 emit_int8((unsigned char)0xFA);
7168 emit_operand(dst, src, 0);
7169 }
7170
7171 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7172 assert(UseAVX > 0, "requires some form of AVX");
7173 InstructionMark im(this);
7174 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7175 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7176 attributes.set_rex_vex_w_reverted();
7177 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7178 emit_int8((unsigned char)0xFB);
7179 emit_operand(dst, src, 0);
7180 }
7181
7182 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
7183 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7184 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7185 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7186 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7187 }
7188
7189 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
7190 assert(VM_Version::supports_sse4_1(), "");
7191 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7192 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7193 emit_int16(0x40, (0xC0 | encode));
7194 }
7195
7196 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
7197 assert(VM_Version::supports_sse2(), "");
7198 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7199 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7200 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7201 }
7202
7203 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7204 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
7205 (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
7206 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
7207 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7208 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7209 emit_int16((unsigned char)0xE4, (0xC0 | encode));
7210 }
7211
7212 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7213 assert(UseAVX > 0, "requires some form of AVX");
7214 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7215 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7216 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7217 }
7218
7219 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7220 assert(UseAVX > 0, "requires some form of AVX");
7221 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7222 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7223 emit_int16(0x40, (0xC0 | encode));
7224 }
7225
7226 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7227 assert(UseAVX > 2, "requires some form of EVEX");
7228 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7229 attributes.set_is_evex_instruction();
7230 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7231 emit_int16(0x40, (0xC0 | encode));
7232 }
7233
7234 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7235 assert(UseAVX > 0, "requires some form of AVX");
7236 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7237 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7238 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7239 }
7240
7241 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7242 assert(UseAVX > 0, "requires some form of AVX");
7243 InstructionMark im(this);
7244 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7245 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7246 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7247 emit_int8((unsigned char)0xD5);
7248 emit_operand(dst, src, 0);
7249 }
7250
7251 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7252 assert(UseAVX > 0, "requires some form of AVX");
7253 InstructionMark im(this);
7254 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7255 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7256 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7257 emit_int8(0x40);
7258 emit_operand(dst, src, 0);
7259 }
7260
7261 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7262 assert(UseAVX > 2, "requires some form of EVEX");
7263 InstructionMark im(this);
7264 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7265 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7266 attributes.set_is_evex_instruction();
7267 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7268 emit_int8(0x40);
7269 emit_operand(dst, src, 0);
7270 }
7271
7272 // Min, max
7273 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
7274 assert(VM_Version::supports_sse4_1(), "");
7275 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7276 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7277 emit_int16(0x38, (0xC0 | encode));
7278 }
7279
7280 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7281 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7282 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7283 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7284 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7285 emit_int16(0x38, (0xC0 | encode));
7286 }
7287
7288 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
7289 assert(VM_Version::supports_sse2(), "");
7290 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7291 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7292 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7293 }
7294
7295 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7296 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7297 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7299 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7300 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7301 }
7302
7303 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
7304 assert(VM_Version::supports_sse4_1(), "");
7305 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7306 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7307 emit_int16(0x39, (0xC0 | encode));
7308 }
7309
7310 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7311 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7312 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7313 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7314 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7315 emit_int16(0x39, (0xC0 | encode));
7316 }
7317
7318 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7319 assert(UseAVX > 2, "requires AVX512F");
7320 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7321 attributes.set_is_evex_instruction();
7322 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7323 emit_int16(0x39, (0xC0 | encode));
7324 }
7325
7326 void Assembler::minps(XMMRegister dst, XMMRegister src) {
7327 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7328 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7329 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7330 emit_int16(0x5D, (0xC0 | encode));
7331 }
7332 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7333 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7334 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7335 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7336 emit_int16(0x5D, (0xC0 | encode));
7337 }
7338
7339 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
7340 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7341 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7342 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7343 emit_int16(0x5D, (0xC0 | encode));
7344 }
7345 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7346 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7347 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7348 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7349 emit_int16(0x5D, (0xC0 | encode));
7350 }
7351
7352 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
7353 assert(VM_Version::supports_sse4_1(), "");
7354 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7355 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7356 emit_int16(0x3C, (0xC0 | encode));
7357 }
7358
7359 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7360 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7361 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7362 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7363 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7364 emit_int16(0x3C, (0xC0 | encode));
7365 }
7366
7367 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
7368 assert(VM_Version::supports_sse2(), "");
7369 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7370 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7371 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7372 }
7373
7374 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7375 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7376 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7377 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7378 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7379 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7380 }
7381
7382 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
7383 assert(VM_Version::supports_sse4_1(), "");
7384 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7385 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7386 emit_int16(0x3D, (0xC0 | encode));
7387 }
7388
7389 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7390 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7391 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7392 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7393 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7394 emit_int16(0x3D, (0xC0 | encode));
7395 }
7396
7397 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7398 assert(UseAVX > 2, "requires AVX512F");
7399 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7400 attributes.set_is_evex_instruction();
7401 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7402 emit_int16(0x3D, (0xC0 | encode));
7403 }
7404
7405 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
7406 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7407 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7408 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7409 emit_int16(0x5F, (0xC0 | encode));
7410 }
7411
7412 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7413 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7414 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7415 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7416 emit_int16(0x5F, (0xC0 | encode));
7417 }
7418
7419 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
7420 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7421 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7422 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7423 emit_int16(0x5F, (0xC0 | encode));
7424 }
7425
7426 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7427 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7428 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7429 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7430 emit_int16(0x5F, (0xC0 | encode));
7431 }
7432
7433 // Shift packed integers left by specified number of bits.
7434 void Assembler::psllw(XMMRegister dst, int shift) {
7435 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7436 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7437 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7438 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7439 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7440 }
7441
7442 void Assembler::pslld(XMMRegister dst, int shift) {
7443 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7444 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7445 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7446 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7447 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7448 }
7449
7450 void Assembler::psllq(XMMRegister dst, int shift) {
7451 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7452 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7453 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7454 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7455 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7456 }
7457
7458 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
7459 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7460 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7461 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7462 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7463 }
7464
7465 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
7466 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7467 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7468 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7469 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7470 }
7471
7472 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
7473 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7474 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7475 attributes.set_rex_vex_w_reverted();
7476 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7477 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7478 }
7479
7480 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7481 assert(UseAVX > 0, "requires some form of AVX");
7482 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7483 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7484 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7485 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7486 }
7487
7488 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7489 assert(UseAVX > 0, "requires some form of AVX");
7490 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7491 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7492 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7493 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7494 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7495 }
7496
7497 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7498 assert(UseAVX > 0, "requires some form of AVX");
7499 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7500 attributes.set_rex_vex_w_reverted();
7501 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7502 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7503 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7504 }
7505
7506 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7507 assert(UseAVX > 0, "requires some form of AVX");
7508 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7509 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7510 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7511 }
7512
7513 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7514 assert(UseAVX > 0, "requires some form of AVX");
7515 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7516 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7517 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7518 }
7519
7520 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7521 assert(UseAVX > 0, "requires some form of AVX");
7522 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7523 attributes.set_rex_vex_w_reverted();
7524 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7525 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7526 }
7527
7528 // Shift packed integers logically right by specified number of bits.
7529 void Assembler::psrlw(XMMRegister dst, int shift) {
7530 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7531 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7532 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7533 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7534 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7535 }
7536
7537 void Assembler::psrld(XMMRegister dst, int shift) {
7538 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7539 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7540 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7541 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7542 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7543 }
7544
7545 void Assembler::psrlq(XMMRegister dst, int shift) {
7546 // Do not confuse it with psrldq SSE2 instruction which
7547 // shifts 128 bit value in xmm register by number of bytes.
7548 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7549 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7550 attributes.set_rex_vex_w_reverted();
7551 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7552 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7553 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7554 }
7555
7556 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
7557 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7558 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7559 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7560 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7561 }
7562
7563 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
7564 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7565 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7566 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7567 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7568 }
7569
7570 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
7571 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7572 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7573 attributes.set_rex_vex_w_reverted();
7574 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7575 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7576 }
7577
7578 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7579 assert(UseAVX > 0, "requires some form of AVX");
7580 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7581 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7582 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7583 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7584 }
7585
7586 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7587 assert(UseAVX > 0, "requires some form of AVX");
7588 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7589 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7590 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7591 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7592 }
7593
7594 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7595 assert(UseAVX > 0, "requires some form of AVX");
7596 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7597 attributes.set_rex_vex_w_reverted();
7598 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7599 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7600 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7601 }
7602
7603 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7604 assert(UseAVX > 0, "requires some form of AVX");
7605 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7606 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7607 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7608 }
7609
7610 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7611 assert(UseAVX > 0, "requires some form of AVX");
7612 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7613 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7614 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7615 }
7616
7617 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7618 assert(UseAVX > 0, "requires some form of AVX");
7619 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7620 attributes.set_rex_vex_w_reverted();
7621 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7622 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7623 }
7624
7625 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7626 assert(VM_Version::supports_avx512bw(), "");
7627 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7628 attributes.set_is_evex_instruction();
7629 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7630 emit_int16(0x10, (0xC0 | encode));
7631 }
7632
7633 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7634 assert(VM_Version::supports_avx512bw(), "");
7635 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7636 attributes.set_is_evex_instruction();
7637 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7638 emit_int16(0x12, (0xC0 | encode));
7639 }
7640
7641 // Shift packed integers arithmetically right by specified number of bits.
7642 void Assembler::psraw(XMMRegister dst, int shift) {
7643 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7644 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7645 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7646 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7647 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7648 }
7649
7650 void Assembler::psrad(XMMRegister dst, int shift) {
7651 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7652 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7653 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
7654 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7655 emit_int8(0x72);
7656 emit_int8((0xC0 | encode));
7657 emit_int8(shift & 0xFF);
7658 }
7659
7660 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
7661 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7662 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7663 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7664 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7665 }
7666
7667 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
7668 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7669 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7670 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7671 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7672 }
7673
7674 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7675 assert(UseAVX > 0, "requires some form of AVX");
7676 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7677 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7678 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7679 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7680 }
7681
7682 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7683 assert(UseAVX > 0, "requires some form of AVX");
7684 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7685 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7686 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7687 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7688 }
7689
7690 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7691 assert(UseAVX > 0, "requires some form of AVX");
7692 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7693 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7694 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7695 }
7696
7697 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7698 assert(UseAVX > 0, "requires some form of AVX");
7699 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7700 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7701 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7702 }
7703
7704 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7705 assert(UseAVX > 2, "requires AVX512");
7706 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7707 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7708 attributes.set_is_evex_instruction();
7709 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7710 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
7711 }
7712
7713 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7714 assert(UseAVX > 2, "requires AVX512");
7715 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7716 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7717 attributes.set_is_evex_instruction();
7718 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7719 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7720 }
7721
7722 // logical operations packed integers
7723 void Assembler::pand(XMMRegister dst, XMMRegister src) {
7724 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7725 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7726 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7727 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7728 }
7729
7730 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7731 assert(UseAVX > 0, "requires some form of AVX");
7732 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7733 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7734 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7735 }
7736
7737 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7738 assert(UseAVX > 0, "requires some form of AVX");
7739 InstructionMark im(this);
7740 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7741 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7742 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7743 emit_int8((unsigned char)0xDB);
7744 emit_operand(dst, src, 0);
7745 }
7746
7747 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7748 assert(VM_Version::supports_evex(), "");
7749 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7750 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7751 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7752 }
7753
7754 //Variable Shift packed integers logically left.
7755 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7756 assert(UseAVX > 1, "requires AVX2");
7757 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7758 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7759 emit_int16(0x47, (0xC0 | encode));
7760 }
7761
7762 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7763 assert(UseAVX > 1, "requires AVX2");
7764 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7765 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7766 emit_int16(0x47, (0xC0 | encode));
7767 }
7768
7769 //Variable Shift packed integers logically right.
7770 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7771 assert(UseAVX > 1, "requires AVX2");
7772 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7773 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7774 emit_int16(0x45, (0xC0 | encode));
7775 }
7776
7777 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7778 assert(UseAVX > 1, "requires AVX2");
7779 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7780 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7781 emit_int16(0x45, (0xC0 | encode));
7782 }
7783
7784 //Variable right Shift arithmetic packed integers .
7785 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7786 assert(UseAVX > 1, "requires AVX2");
7787 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7788 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7789 emit_int16(0x46, (0xC0 | encode));
7790 }
7791
7792 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7793 assert(VM_Version::supports_avx512bw(), "");
7794 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7795 attributes.set_is_evex_instruction();
7796 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7797 emit_int16(0x11, (0xC0 | encode));
7798 }
7799
7800 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7801 assert(UseAVX > 2, "requires AVX512");
7802 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
7803 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7804 attributes.set_is_evex_instruction();
7805 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7806 emit_int16(0x46, (0xC0 | encode));
7807 }
7808
7809 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7810 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7811 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7812 attributes.set_is_evex_instruction();
7813 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7814 emit_int16(0x71, (0xC0 | encode));
7815 }
7816
7817 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7818 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7819 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7820 attributes.set_is_evex_instruction();
7821 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7822 emit_int16(0x73, (0xC0 | encode));
7823 }
7824
7825 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
7826 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7827 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7828 attributes.set_rex_vex_w_reverted();
7829 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7830 emit_int16((unsigned char)0xDF, (0xC0 | encode));
7831 }
7832
7833 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7834 assert(UseAVX > 0, "requires some form of AVX");
7835 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7837 emit_int16((unsigned char)0xDF, (0xC0 | encode));
7838 }
7839
7840 void Assembler::por(XMMRegister dst, XMMRegister src) {
7841 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7842 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7843 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7844 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7845 }
7846
7847 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7848 assert(UseAVX > 0, "requires some form of AVX");
7849 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7850 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7851 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7852 }
7853
7854 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7855 assert(UseAVX > 0, "requires some form of AVX");
7856 InstructionMark im(this);
7857 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7858 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7859 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7860 emit_int8((unsigned char)0xEB);
7861 emit_operand(dst, src, 0);
7862 }
7863
7864 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7865 assert(VM_Version::supports_evex(), "");
7866 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7867 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7868 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7869 }
7870
7871
7872 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7873 assert(VM_Version::supports_evex(), "");
7874 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
7875 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7876 attributes.set_is_evex_instruction();
7877 attributes.set_embedded_opmask_register_specifier(mask);
7878 if (merge) {
7879 attributes.reset_is_clear_context();
7880 }
7881 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7882 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7883 }
7884
7885 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7886 assert(VM_Version::supports_evex(), "");
7887 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
7888 InstructionMark im(this);
7889 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7890 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7891 attributes.set_is_evex_instruction();
7892 attributes.set_embedded_opmask_register_specifier(mask);
7893 if (merge) {
7894 attributes.reset_is_clear_context();
7895 }
7896 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7897 emit_int8((unsigned char)0xEB);
7898 emit_operand(dst, src, 0);
7899 }
7900
7901 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
7902 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7903 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7904 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7905 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7906 }
7907
7908 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7909 assert(UseAVX > 0, "requires some form of AVX");
7910 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7911 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
7912 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
7913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7914 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7915 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7916 }
7917
7918 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7919 assert(UseAVX > 0, "requires some form of AVX");
7920 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7921 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
7922 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
7923 InstructionMark im(this);
7924 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7925 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7926 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7927 emit_int8((unsigned char)0xEF);
7928 emit_operand(dst, src, 0);
7929 }
7930
7931 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7932 assert(UseAVX > 2, "requires some form of EVEX");
7933 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7934 attributes.set_rex_vex_w_reverted();
7935 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7936 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7937 }
7938
7939 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7940 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
7941 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7942 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7943 attributes.set_is_evex_instruction();
7944 attributes.set_embedded_opmask_register_specifier(mask);
7945 if (merge) {
7946 attributes.reset_is_clear_context();
7947 }
7948 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7949 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7950 }
7951
7952 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7953 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7954 InstructionMark im(this);
7955 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7956 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7957 attributes.set_is_evex_instruction();
7958 attributes.set_embedded_opmask_register_specifier(mask);
7959 if (merge) {
7960 attributes.reset_is_clear_context();
7961 }
7962 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7963 emit_int8((unsigned char)0xEF);
7964 emit_operand(dst, src, 0);
7965 }
7966
7967 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7968 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
7969 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7970 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7971 attributes.set_is_evex_instruction();
7972 attributes.set_embedded_opmask_register_specifier(mask);
7973 if (merge) {
7974 attributes.reset_is_clear_context();
7975 }
7976 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7977 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7978 }
7979
7980 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7981 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7982 InstructionMark im(this);
7983 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7984 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7985 attributes.set_is_evex_instruction();
7986 attributes.set_embedded_opmask_register_specifier(mask);
7987 if (merge) {
7988 attributes.reset_is_clear_context();
7989 }
7990 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7991 emit_int8((unsigned char)0xEF);
7992 emit_operand(dst, src, 0);
7993 }
7994
7995 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7996 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7997 InstructionMark im(this);
7998 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7999 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8000 attributes.set_is_evex_instruction();
8001 attributes.set_embedded_opmask_register_specifier(mask);
8002 if (merge) {
8003 attributes.reset_is_clear_context();
8004 }
8005 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8006 emit_int8((unsigned char)0xDB);
8007 emit_operand(dst, src, 0);
8008 }
8009
8010 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8011 assert(VM_Version::supports_evex(), "");
8012 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8013 attributes.set_is_evex_instruction();
8014 attributes.set_embedded_opmask_register_specifier(mask);
8015 if (merge) {
8016 attributes.reset_is_clear_context();
8017 }
8018 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8019 emit_int16((unsigned char)0xDB, (0xC0 | encode));
8020 }
8021
8022 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8023 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8024 InstructionMark im(this);
8025 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8026 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8027 attributes.set_is_evex_instruction();
8028 attributes.set_embedded_opmask_register_specifier(mask);
8029 if (merge) {
8030 attributes.reset_is_clear_context();
8031 }
8032 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8033 emit_int8((unsigned char)0xDB);
8034 emit_operand(dst, src, 0);
8035 }
8036
8037 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8038 assert(VM_Version::supports_evex(), "");
8039 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8040 attributes.set_is_evex_instruction();
8041 attributes.set_embedded_opmask_register_specifier(mask);
8042 if (merge) {
8043 attributes.reset_is_clear_context();
8044 }
8045 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8046 emit_int16((unsigned char)0xEB, (0xC0 | encode));
8047 }
8048
8049 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8050 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8051 InstructionMark im(this);
8052 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8053 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8054 attributes.set_is_evex_instruction();
8055 attributes.set_embedded_opmask_register_specifier(mask);
8056 if (merge) {
8057 attributes.reset_is_clear_context();
8058 }
8059 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8060 emit_int8((unsigned char)0xEB);
8061 emit_operand(dst, src, 0);
8062 }
8063
8064 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8065 assert(VM_Version::supports_evex(), "requires EVEX support");
8066 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8067 attributes.set_is_evex_instruction();
8068 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8069 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8070 }
8071
8072 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8073 assert(VM_Version::supports_evex(), "requires EVEX support");
8074 assert(dst != xnoreg, "sanity");
8075 InstructionMark im(this);
8076 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8077 attributes.set_is_evex_instruction();
8078 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8079 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8080 emit_int8((unsigned char)0xEF);
8081 emit_operand(dst, src, 0);
8082 }
8083
8084 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8085 assert(VM_Version::supports_evex(), "requires EVEX support");
8086 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8088 attributes.set_is_evex_instruction();
8089 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8090 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8091 }
8092
8093 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8094 assert(VM_Version::supports_evex(), "requires EVEX support");
8095 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8096 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8097 attributes.set_is_evex_instruction();
8098 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8099 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8100 }
8101
8102 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8103 assert(VM_Version::supports_evex(), "requires EVEX support");
8104 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8105 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8106 attributes.set_is_evex_instruction();
8107 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8108 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8109 }
8110
8111 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8112 assert(VM_Version::supports_evex(), "requires EVEX support");
8113 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8114 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8115 attributes.set_is_evex_instruction();
8116 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8117 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8118 }
8119
8120 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8121 assert(VM_Version::supports_evex(), "requires EVEX support");
8122 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8123 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8124 attributes.set_is_evex_instruction();
8125 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8126 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8127 }
8128
8129 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8130 assert(VM_Version::supports_evex(), "requires EVEX support");
8131 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8132 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8133 attributes.set_is_evex_instruction();
8134 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8135 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8136 }
8137
8138 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8139 assert(VM_Version::supports_evex(), "requires EVEX support");
8140 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8141 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8142 attributes.set_is_evex_instruction();
8143 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8144 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8145 }
8146
8147 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8148 assert(VM_Version::supports_evex(), "requires EVEX support");
8149 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8150 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8151 attributes.set_is_evex_instruction();
8152 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8153 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8154 }
8155
8156 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8157 assert(VM_Version::supports_avx512cd(), "");
8158 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8159 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8160 attributes.set_is_evex_instruction();
8161 attributes.set_embedded_opmask_register_specifier(mask);
8162 if (merge) {
8163 attributes.reset_is_clear_context();
8164 }
8165 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8166 emit_int16(0x44, (0xC0 | encode));
8167 }
8168
8169 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8170 assert(VM_Version::supports_avx512cd(), "");
8171 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8172 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8173 attributes.set_is_evex_instruction();
8174 attributes.set_embedded_opmask_register_specifier(mask);
8175 if (merge) {
8176 attributes.reset_is_clear_context();
8177 }
8178 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8179 emit_int16(0x44, (0xC0 | encode));
8180 }
8181
8182 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8183 assert(VM_Version::supports_evex(), "requires EVEX support");
8184 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8185 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8186 attributes.set_is_evex_instruction();
8187 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8188 emit_int8(0x25);
8189 emit_int8((unsigned char)(0xC0 | encode));
8190 emit_int8(imm8);
8191 }
8192
8193 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
8194 assert(VM_Version::supports_evex(), "requires EVEX support");
8195 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8196 assert(dst != xnoreg, "sanity");
8197 InstructionMark im(this);
8198 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8199 attributes.set_is_evex_instruction();
8200 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8201 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8202 emit_int8(0x25);
8203 emit_operand(dst, src3, 1);
8204 emit_int8(imm8);
8205 }
8206
8207 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8208 assert(VM_Version::supports_evex(), "requires EVEX support");
8209 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8210 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8211 attributes.set_is_evex_instruction();
8212 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8213 emit_int8(0x25);
8214 emit_int8((unsigned char)(0xC0 | encode));
8215 emit_int8(imm8);
8216 }
8217
8218 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8219 assert(VM_Version::supports_evex(), "");
8220 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8221 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8222 attributes.set_is_evex_instruction();
8223 attributes.set_embedded_opmask_register_specifier(mask);
8224 if (merge) {
8225 attributes.reset_is_clear_context();
8226 }
8227 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8228 emit_int16((unsigned char)0x88, (0xC0 | encode));
8229 }
8230
8231 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8232 assert(VM_Version::supports_evex(), "");
8233 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8234 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8235 attributes.set_is_evex_instruction();
8236 attributes.set_embedded_opmask_register_specifier(mask);
8237 if (merge) {
8238 attributes.reset_is_clear_context();
8239 }
8240 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8241 emit_int16((unsigned char)0x88, (0xC0 | encode));
8242 }
8243
8244 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8245 assert(VM_Version::supports_avx512_vbmi2(), "");
8246 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8247 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8248 attributes.set_is_evex_instruction();
8249 attributes.set_embedded_opmask_register_specifier(mask);
8250 if (merge) {
8251 attributes.reset_is_clear_context();
8252 }
8253 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8254 emit_int16(0x62, (0xC0 | encode));
8255 }
8256
8257 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8258 assert(VM_Version::supports_avx512_vbmi2(), "");
8259 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8260 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8261 attributes.set_is_evex_instruction();
8262 attributes.set_embedded_opmask_register_specifier(mask);
8263 if (merge) {
8264 attributes.reset_is_clear_context();
8265 }
8266 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8267 emit_int16(0x62, (0xC0 | encode));
8268 }
8269
8270 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8271 assert(VM_Version::supports_evex(), "");
8272 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8273 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8274 attributes.set_is_evex_instruction();
8275 attributes.set_embedded_opmask_register_specifier(mask);
8276 if (merge) {
8277 attributes.reset_is_clear_context();
8278 }
8279 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8280 emit_int16((unsigned char)0x89, (0xC0 | encode));
8281 }
8282
8283 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8284 assert(VM_Version::supports_evex(), "");
8285 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8286 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8287 attributes.set_is_evex_instruction();
8288 attributes.set_embedded_opmask_register_specifier(mask);
8289 if (merge) {
8290 attributes.reset_is_clear_context();
8291 }
8292 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8293 emit_int16((unsigned char)0x89, (0xC0 | encode));
8294 }
8295
8296 // vinserti forms
8297
8298 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8299 assert(VM_Version::supports_avx2(), "");
8300 assert(imm8 <= 0x01, "imm8: %u", imm8);
8301 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8302 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8303 // last byte:
8304 // 0x00 - insert into lower 128 bits
8305 // 0x01 - insert into upper 128 bits
8306 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
8307 }
8308
8309 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8310 assert(VM_Version::supports_avx2(), "");
8311 assert(dst != xnoreg, "sanity");
8312 assert(imm8 <= 0x01, "imm8: %u", imm8);
8313 InstructionMark im(this);
8314 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8315 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8316 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8317 emit_int8(0x38);
8318 emit_operand(dst, src, 1);
8319 // 0x00 - insert into lower 128 bits
8320 // 0x01 - insert into upper 128 bits
8321 emit_int8(imm8 & 0x01);
8322 }
8323
8324 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8325 assert(VM_Version::supports_evex(), "");
8326 assert(imm8 <= 0x03, "imm8: %u", imm8);
8327 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8328 attributes.set_is_evex_instruction();
8329 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8330 // imm8:
8331 // 0x00 - insert into q0 128 bits (0..127)
8332 // 0x01 - insert into q1 128 bits (128..255)
8333 // 0x02 - insert into q2 128 bits (256..383)
8334 // 0x03 - insert into q3 128 bits (384..511)
8335 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
8336 }
8337
8338 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8339 assert(VM_Version::supports_evex(), "");
8340 assert(dst != xnoreg, "sanity");
8341 assert(imm8 <= 0x03, "imm8: %u", imm8);
8342 InstructionMark im(this);
8343 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8344 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8345 attributes.set_is_evex_instruction();
8346 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8347 emit_int8(0x18);
8348 emit_operand(dst, src, 1);
8349 // 0x00 - insert into q0 128 bits (0..127)
8350 // 0x01 - insert into q1 128 bits (128..255)
8351 // 0x02 - insert into q2 128 bits (256..383)
8352 // 0x03 - insert into q3 128 bits (384..511)
8353 emit_int8(imm8 & 0x03);
8354 }
8355
8356 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8357 assert(VM_Version::supports_evex(), "");
8358 assert(imm8 <= 0x01, "imm8: %u", imm8);
8359 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8360 attributes.set_is_evex_instruction();
8361 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8362 //imm8:
8363 // 0x00 - insert into lower 256 bits
8364 // 0x01 - insert into upper 256 bits
8365 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
8366 }
8367
8368
8369 // vinsertf forms
8370
8371 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8372 assert(VM_Version::supports_avx(), "");
8373 assert(imm8 <= 0x01, "imm8: %u", imm8);
8374 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8375 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8376 // imm8:
8377 // 0x00 - insert into lower 128 bits
8378 // 0x01 - insert into upper 128 bits
8379 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
8380 }
8381
8382 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8383 assert(VM_Version::supports_avx(), "");
8384 assert(dst != xnoreg, "sanity");
8385 assert(imm8 <= 0x01, "imm8: %u", imm8);
8386 InstructionMark im(this);
8387 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8388 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8389 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8390 emit_int8(0x18);
8391 emit_operand(dst, src, 1);
8392 // 0x00 - insert into lower 128 bits
8393 // 0x01 - insert into upper 128 bits
8394 emit_int8(imm8 & 0x01);
8395 }
8396
8397 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8398 assert(VM_Version::supports_evex(), "");
8399 assert(imm8 <= 0x03, "imm8: %u", imm8);
8400 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8401 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8402 // imm8:
8403 // 0x00 - insert into q0 128 bits (0..127)
8404 // 0x01 - insert into q1 128 bits (128..255)
8405 // 0x02 - insert into q0 128 bits (256..383)
8406 // 0x03 - insert into q1 128 bits (384..512)
8407 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
8408 }
8409
8410 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8411 assert(VM_Version::supports_evex(), "");
8412 assert(dst != xnoreg, "sanity");
8413 assert(imm8 <= 0x03, "imm8: %u", imm8);
8414 InstructionMark im(this);
8415 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8416 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8417 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8418 emit_int8(0x18);
8419 emit_operand(dst, src, 1);
8420 // 0x00 - insert into q0 128 bits (0..127)
8421 // 0x01 - insert into q1 128 bits (128..255)
8422 // 0x02 - insert into q0 128 bits (256..383)
8423 // 0x03 - insert into q1 128 bits (384..512)
8424 emit_int8(imm8 & 0x03);
8425 }
8426
8427 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8428 assert(VM_Version::supports_evex(), "");
8429 assert(imm8 <= 0x01, "imm8: %u", imm8);
8430 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8431 attributes.set_is_evex_instruction();
8432 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8433 // imm8:
8434 // 0x00 - insert into lower 256 bits
8435 // 0x01 - insert into upper 256 bits
8436 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
8437 }
8438
8439 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8440 assert(VM_Version::supports_evex(), "");
8441 assert(dst != xnoreg, "sanity");
8442 assert(imm8 <= 0x01, "imm8: %u", imm8);
8443 InstructionMark im(this);
8444 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8445 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8446 attributes.set_is_evex_instruction();
8447 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8448 emit_int8(0x1A);
8449 emit_operand(dst, src, 1);
8450 // 0x00 - insert into lower 256 bits
8451 // 0x01 - insert into upper 256 bits
8452 emit_int8(imm8 & 0x01);
8453 }
8454
8455
8456 // vextracti forms
8457
8458 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8459 assert(VM_Version::supports_avx2(), "");
8460 assert(imm8 <= 0x01, "imm8: %u", imm8);
8461 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8462 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8463 // imm8:
8464 // 0x00 - extract from lower 128 bits
8465 // 0x01 - extract from upper 128 bits
8466 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
8467 }
8468
8469 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
8470 assert(VM_Version::supports_avx2(), "");
8471 assert(src != xnoreg, "sanity");
8472 assert(imm8 <= 0x01, "imm8: %u", imm8);
8473 InstructionMark im(this);
8474 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8475 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8476 attributes.reset_is_clear_context();
8477 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8478 emit_int8(0x39);
8479 emit_operand(src, dst, 1);
8480 // 0x00 - extract from lower 128 bits
8481 // 0x01 - extract from upper 128 bits
8482 emit_int8(imm8 & 0x01);
8483 }
8484
8485 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8486 assert(VM_Version::supports_evex(), "");
8487 assert(imm8 <= 0x03, "imm8: %u", imm8);
8488 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8489 attributes.set_is_evex_instruction();
8490 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8491 // imm8:
8492 // 0x00 - extract from bits 127:0
8493 // 0x01 - extract from bits 255:128
8494 // 0x02 - extract from bits 383:256
8495 // 0x03 - extract from bits 511:384
8496 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8497 }
8498
8499 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
8500 assert(VM_Version::supports_evex(), "");
8501 assert(src != xnoreg, "sanity");
8502 assert(imm8 <= 0x03, "imm8: %u", imm8);
8503 InstructionMark im(this);
8504 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8505 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8506 attributes.reset_is_clear_context();
8507 attributes.set_is_evex_instruction();
8508 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8509 emit_int8(0x39);
8510 emit_operand(src, dst, 1);
8511 // 0x00 - extract from bits 127:0
8512 // 0x01 - extract from bits 255:128
8513 // 0x02 - extract from bits 383:256
8514 // 0x03 - extract from bits 511:384
8515 emit_int8(imm8 & 0x03);
8516 }
8517
8518 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8519 assert(VM_Version::supports_avx512dq(), "");
8520 assert(imm8 <= 0x03, "imm8: %u", imm8);
8521 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8522 attributes.set_is_evex_instruction();
8523 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8524 // imm8:
8525 // 0x00 - extract from bits 127:0
8526 // 0x01 - extract from bits 255:128
8527 // 0x02 - extract from bits 383:256
8528 // 0x03 - extract from bits 511:384
8529 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8530 }
8531
8532 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8533 assert(VM_Version::supports_evex(), "");
8534 assert(imm8 <= 0x01, "imm8: %u", imm8);
8535 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8536 attributes.set_is_evex_instruction();
8537 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8538 // imm8:
8539 // 0x00 - extract from lower 256 bits
8540 // 0x01 - extract from upper 256 bits
8541 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
8542 }
8543
8544 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
8545 assert(VM_Version::supports_evex(), "");
8546 assert(src != xnoreg, "sanity");
8547 assert(imm8 <= 0x01, "imm8: %u", imm8);
8548 InstructionMark im(this);
8549 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8550 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8551 attributes.reset_is_clear_context();
8552 attributes.set_is_evex_instruction();
8553 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8554 emit_int8(0x38);
8555 emit_operand(src, dst, 1);
8556 // 0x00 - extract from lower 256 bits
8557 // 0x01 - extract from upper 256 bits
8558 emit_int8(imm8 & 0x01);
8559 }
8560 // vextractf forms
8561
8562 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8563 assert(VM_Version::supports_avx(), "");
8564 assert(imm8 <= 0x01, "imm8: %u", imm8);
8565 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8566 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8567 // imm8:
8568 // 0x00 - extract from lower 128 bits
8569 // 0x01 - extract from upper 128 bits
8570 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
8571 }
8572
8573 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
8574 assert(VM_Version::supports_avx(), "");
8575 assert(src != xnoreg, "sanity");
8576 assert(imm8 <= 0x01, "imm8: %u", imm8);
8577 InstructionMark im(this);
8578 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8579 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8580 attributes.reset_is_clear_context();
8581 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8582 emit_int8(0x19);
8583 emit_operand(src, dst, 1);
8584 // 0x00 - extract from lower 128 bits
8585 // 0x01 - extract from upper 128 bits
8586 emit_int8(imm8 & 0x01);
8587 }
8588
8589 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8590 assert(VM_Version::supports_evex(), "");
8591 assert(imm8 <= 0x03, "imm8: %u", imm8);
8592 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8593 attributes.set_is_evex_instruction();
8594 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8595 // imm8:
8596 // 0x00 - extract from bits 127:0
8597 // 0x01 - extract from bits 255:128
8598 // 0x02 - extract from bits 383:256
8599 // 0x03 - extract from bits 511:384
8600 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8601 }
8602
8603 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
8604 assert(VM_Version::supports_evex(), "");
8605 assert(src != xnoreg, "sanity");
8606 assert(imm8 <= 0x03, "imm8: %u", imm8);
8607 InstructionMark im(this);
8608 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8609 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8610 attributes.reset_is_clear_context();
8611 attributes.set_is_evex_instruction();
8612 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8613 emit_int8(0x19);
8614 emit_operand(src, dst, 1);
8615 // 0x00 - extract from bits 127:0
8616 // 0x01 - extract from bits 255:128
8617 // 0x02 - extract from bits 383:256
8618 // 0x03 - extract from bits 511:384
8619 emit_int8(imm8 & 0x03);
8620 }
8621
8622 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8623 assert(VM_Version::supports_avx512dq(), "");
8624 assert(imm8 <= 0x03, "imm8: %u", imm8);
8625 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8626 attributes.set_is_evex_instruction();
8627 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8628 // imm8:
8629 // 0x00 - extract from bits 127:0
8630 // 0x01 - extract from bits 255:128
8631 // 0x02 - extract from bits 383:256
8632 // 0x03 - extract from bits 511:384
8633 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8634 }
8635
8636 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8637 assert(VM_Version::supports_evex(), "");
8638 assert(imm8 <= 0x01, "imm8: %u", imm8);
8639 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8640 attributes.set_is_evex_instruction();
8641 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8642 // imm8:
8643 // 0x00 - extract from lower 256 bits
8644 // 0x01 - extract from upper 256 bits
8645 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
8646 }
8647
8648 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
8649 assert(VM_Version::supports_evex(), "");
8650 assert(src != xnoreg, "sanity");
8651 assert(imm8 <= 0x01, "imm8: %u", imm8);
8652 InstructionMark im(this);
8653 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8654 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
8655 attributes.reset_is_clear_context();
8656 attributes.set_is_evex_instruction();
8657 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8658 emit_int8(0x1B);
8659 emit_operand(src, dst, 1);
8660 // 0x00 - extract from lower 256 bits
8661 // 0x01 - extract from upper 256 bits
8662 emit_int8(imm8 & 0x01);
8663 }
8664
8665 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8666 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
8667 assert(VM_Version::supports_avx2(), "");
8668 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8669 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8670 emit_int16(0x78, (0xC0 | encode));
8671 }
8672
8673 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
8674 assert(VM_Version::supports_avx2(), "");
8675 assert(dst != xnoreg, "sanity");
8676 InstructionMark im(this);
8677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8678 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
8679 // swap src<->dst for encoding
8680 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8681 emit_int8(0x78);
8682 emit_operand(dst, src, 0);
8683 }
8684
8685 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8686 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
8687 assert(VM_Version::supports_avx2(), "");
8688 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8689 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8690 emit_int16(0x79, (0xC0 | encode));
8691 }
8692
8693 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
8694 assert(VM_Version::supports_avx2(), "");
8695 assert(dst != xnoreg, "sanity");
8696 InstructionMark im(this);
8697 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8698 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
8699 // swap src<->dst for encoding
8700 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8701 emit_int8(0x79);
8702 emit_operand(dst, src, 0);
8703 }
8704
8705 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8706 assert(UseAVX > 0, "requires some form of AVX");
8707 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8708 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8709 emit_int16((unsigned char)0xF6, (0xC0 | encode));
8710 }
8711
8712 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8713 assert(UseAVX > 0, "requires some form of AVX");
8714 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8715 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8716 emit_int16(0x69, (0xC0 | encode));
8717 }
8718
8719 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8720 assert(UseAVX > 0, "requires some form of AVX");
8721 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8722 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8723 emit_int16(0x61, (0xC0 | encode));
8724 }
8725
8726 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8727 assert(UseAVX > 0, "requires some form of AVX");
8728 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8729 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8730 emit_int16(0x6A, (0xC0 | encode));
8731 }
8732
8733 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8734 assert(UseAVX > 0, "requires some form of AVX");
8735 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8736 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8737 emit_int16(0x62, (0xC0 | encode));
8738 }
8739
8740 // xmm/mem sourced byte/word/dword/qword replicate
8741 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8742 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8743 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8744 attributes.set_is_evex_instruction();
8745 attributes.set_embedded_opmask_register_specifier(mask);
8746 if (merge) {
8747 attributes.reset_is_clear_context();
8748 }
8749 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8750 emit_int16((unsigned char)0xFC, (0xC0 | encode));
8751 }
8752
8753 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8754 InstructionMark im(this);
8755 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8756 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8757 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8758 attributes.set_is_evex_instruction();
8759 attributes.set_embedded_opmask_register_specifier(mask);
8760 if (merge) {
8761 attributes.reset_is_clear_context();
8762 }
8763 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8764 emit_int8((unsigned char)0xFC);
8765 emit_operand(dst, src, 0);
8766 }
8767
8768 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8769 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8770 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8771 attributes.set_is_evex_instruction();
8772 attributes.set_embedded_opmask_register_specifier(mask);
8773 if (merge) {
8774 attributes.reset_is_clear_context();
8775 }
8776 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8777 emit_int16((unsigned char)0xFD, (0xC0 | encode));
8778 }
8779
8780 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8781 InstructionMark im(this);
8782 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8783 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8784 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8785 attributes.set_is_evex_instruction();
8786 attributes.set_embedded_opmask_register_specifier(mask);
8787 if (merge) {
8788 attributes.reset_is_clear_context();
8789 }
8790 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8791 emit_int8((unsigned char)0xFD);
8792 emit_operand(dst, src, 0);
8793 }
8794
8795 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8796 assert(VM_Version::supports_evex(), "");
8797 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8798 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8799 attributes.set_is_evex_instruction();
8800 attributes.set_embedded_opmask_register_specifier(mask);
8801 if (merge) {
8802 attributes.reset_is_clear_context();
8803 }
8804 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8805 emit_int16((unsigned char)0xFE, (0xC0 | encode));
8806 }
8807
8808 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8809 InstructionMark im(this);
8810 assert(VM_Version::supports_evex(), "");
8811 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8812 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8813 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8814 attributes.set_is_evex_instruction();
8815 attributes.set_embedded_opmask_register_specifier(mask);
8816 if (merge) {
8817 attributes.reset_is_clear_context();
8818 }
8819 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8820 emit_int8((unsigned char)0xFE);
8821 emit_operand(dst, src, 0);
8822 }
8823
8824 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8825 assert(VM_Version::supports_evex(), "");
8826 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8827 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8828 attributes.set_is_evex_instruction();
8829 attributes.set_embedded_opmask_register_specifier(mask);
8830 if (merge) {
8831 attributes.reset_is_clear_context();
8832 }
8833 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8834 emit_int16((unsigned char)0xD4, (0xC0 | encode));
8835 }
8836
8837 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8838 InstructionMark im(this);
8839 assert(VM_Version::supports_evex(), "");
8840 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8841 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8842 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8843 attributes.set_is_evex_instruction();
8844 attributes.set_embedded_opmask_register_specifier(mask);
8845 if (merge) {
8846 attributes.reset_is_clear_context();
8847 }
8848 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8849 emit_int8((unsigned char)0xD4);
8850 emit_operand(dst, src, 0);
8851 }
8852
8853 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8854 assert(VM_Version::supports_evex(), "");
8855 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8857 attributes.set_is_evex_instruction();
8858 attributes.set_embedded_opmask_register_specifier(mask);
8859 if (merge) {
8860 attributes.reset_is_clear_context();
8861 }
8862 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8863 emit_int16(0x58, (0xC0 | encode));
8864 }
8865
8866 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8867 InstructionMark im(this);
8868 assert(VM_Version::supports_evex(), "");
8869 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8870 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8871 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8872 attributes.set_is_evex_instruction();
8873 attributes.set_embedded_opmask_register_specifier(mask);
8874 if (merge) {
8875 attributes.reset_is_clear_context();
8876 }
8877 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8878 emit_int8(0x58);
8879 emit_operand(dst, src, 0);
8880 }
8881
8882 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8883 assert(VM_Version::supports_evex(), "");
8884 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8885 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8886 attributes.set_is_evex_instruction();
8887 attributes.set_embedded_opmask_register_specifier(mask);
8888 if (merge) {
8889 attributes.reset_is_clear_context();
8890 }
8891 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8892 emit_int16(0x58, (0xC0 | encode));
8893 }
8894
8895 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8896 InstructionMark im(this);
8897 assert(VM_Version::supports_evex(), "");
8898 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8899 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8900 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8901 attributes.set_is_evex_instruction();
8902 attributes.set_embedded_opmask_register_specifier(mask);
8903 if (merge) {
8904 attributes.reset_is_clear_context();
8905 }
8906 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8907 emit_int8(0x58);
8908 emit_operand(dst, src, 0);
8909 }
8910
8911 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8912 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8914 attributes.set_is_evex_instruction();
8915 attributes.set_embedded_opmask_register_specifier(mask);
8916 if (merge) {
8917 attributes.reset_is_clear_context();
8918 }
8919 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8920 emit_int16((unsigned char)0xF8, (0xC0 | encode));
8921 }
8922
8923 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8924 InstructionMark im(this);
8925 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8926 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8927 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8928 attributes.set_is_evex_instruction();
8929 attributes.set_embedded_opmask_register_specifier(mask);
8930 if (merge) {
8931 attributes.reset_is_clear_context();
8932 }
8933 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8934 emit_int8((unsigned char)0xF8);
8935 emit_operand(dst, src, 0);
8936 }
8937
8938 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8939 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8940 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8941 attributes.set_is_evex_instruction();
8942 attributes.set_embedded_opmask_register_specifier(mask);
8943 if (merge) {
8944 attributes.reset_is_clear_context();
8945 }
8946 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8947 emit_int16((unsigned char)0xF9, (0xC0 | encode));
8948 }
8949
8950 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8951 InstructionMark im(this);
8952 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8953 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8954 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8955 attributes.set_is_evex_instruction();
8956 attributes.set_embedded_opmask_register_specifier(mask);
8957 if (merge) {
8958 attributes.reset_is_clear_context();
8959 }
8960 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8961 emit_int8((unsigned char)0xF9);
8962 emit_operand(dst, src, 0);
8963 }
8964
8965 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8966 assert(VM_Version::supports_evex(), "");
8967 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8968 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8969 attributes.set_is_evex_instruction();
8970 attributes.set_embedded_opmask_register_specifier(mask);
8971 if (merge) {
8972 attributes.reset_is_clear_context();
8973 }
8974 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8975 emit_int16((unsigned char)0xFA, (0xC0 | encode));
8976 }
8977
8978 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8979 InstructionMark im(this);
8980 assert(VM_Version::supports_evex(), "");
8981 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8982 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8983 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8984 attributes.set_is_evex_instruction();
8985 attributes.set_embedded_opmask_register_specifier(mask);
8986 if (merge) {
8987 attributes.reset_is_clear_context();
8988 }
8989 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8990 emit_int8((unsigned char)0xFA);
8991 emit_operand(dst, src, 0);
8992 }
8993
8994 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8995 assert(VM_Version::supports_evex(), "");
8996 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8997 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8998 attributes.set_is_evex_instruction();
8999 attributes.set_embedded_opmask_register_specifier(mask);
9000 if (merge) {
9001 attributes.reset_is_clear_context();
9002 }
9003 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9004 emit_int16((unsigned char)0xFB, (0xC0 | encode));
9005 }
9006
9007 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9008 InstructionMark im(this);
9009 assert(VM_Version::supports_evex(), "");
9010 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9011 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9012 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9013 attributes.set_is_evex_instruction();
9014 attributes.set_embedded_opmask_register_specifier(mask);
9015 if (merge) {
9016 attributes.reset_is_clear_context();
9017 }
9018 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9019 emit_int8((unsigned char)0xFB);
9020 emit_operand(dst, src, 0);
9021 }
9022
9023 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9024 assert(VM_Version::supports_evex(), "");
9025 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9026 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9027 attributes.set_is_evex_instruction();
9028 attributes.set_embedded_opmask_register_specifier(mask);
9029 if (merge) {
9030 attributes.reset_is_clear_context();
9031 }
9032 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9033 emit_int16(0x5C, (0xC0 | encode));
9034 }
9035
9036 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9037 InstructionMark im(this);
9038 assert(VM_Version::supports_evex(), "");
9039 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9040 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9041 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9042 attributes.set_is_evex_instruction();
9043 attributes.set_embedded_opmask_register_specifier(mask);
9044 if (merge) {
9045 attributes.reset_is_clear_context();
9046 }
9047 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9048 emit_int8(0x5C);
9049 emit_operand(dst, src, 0);
9050 }
9051
9052 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9053 assert(VM_Version::supports_evex(), "");
9054 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9055 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9056 attributes.set_is_evex_instruction();
9057 attributes.set_embedded_opmask_register_specifier(mask);
9058 if (merge) {
9059 attributes.reset_is_clear_context();
9060 }
9061 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9062 emit_int16(0x5C, (0xC0 | encode));
9063 }
9064
9065 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9066 InstructionMark im(this);
9067 assert(VM_Version::supports_evex(), "");
9068 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9069 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9070 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9071 attributes.set_is_evex_instruction();
9072 attributes.set_embedded_opmask_register_specifier(mask);
9073 if (merge) {
9074 attributes.reset_is_clear_context();
9075 }
9076 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9077 emit_int8(0x5C);
9078 emit_operand(dst, src, 0);
9079 }
9080
9081 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9082 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9083 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9084 attributes.set_is_evex_instruction();
9085 attributes.set_embedded_opmask_register_specifier(mask);
9086 if (merge) {
9087 attributes.reset_is_clear_context();
9088 }
9089 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9090 emit_int16((unsigned char)0xD5, (0xC0 | encode));
9091 }
9092
9093 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9094 InstructionMark im(this);
9095 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9096 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9097 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9098 attributes.set_is_evex_instruction();
9099 attributes.set_embedded_opmask_register_specifier(mask);
9100 if (merge) {
9101 attributes.reset_is_clear_context();
9102 }
9103 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9104 emit_int8((unsigned char)0xD5);
9105 emit_operand(dst, src, 0);
9106 }
9107
9108 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9109 assert(VM_Version::supports_evex(), "");
9110 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9111 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9112 attributes.set_is_evex_instruction();
9113 attributes.set_embedded_opmask_register_specifier(mask);
9114 if (merge) {
9115 attributes.reset_is_clear_context();
9116 }
9117 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9118 emit_int16(0x40, (0xC0 | encode));
9119 }
9120
9121 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9122 InstructionMark im(this);
9123 assert(VM_Version::supports_evex(), "");
9124 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9125 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9126 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9127 attributes.set_is_evex_instruction();
9128 attributes.set_embedded_opmask_register_specifier(mask);
9129 if (merge) {
9130 attributes.reset_is_clear_context();
9131 }
9132 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9133 emit_int8(0x40);
9134 emit_operand(dst, src, 0);
9135 }
9136
9137 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9138 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9139 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9140 attributes.set_is_evex_instruction();
9141 attributes.set_embedded_opmask_register_specifier(mask);
9142 if (merge) {
9143 attributes.reset_is_clear_context();
9144 }
9145 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9146 emit_int16(0x40, (0xC0 | encode));
9147 }
9148
9149 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9150 InstructionMark im(this);
9151 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9152 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9153 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9154 attributes.set_is_evex_instruction();
9155 attributes.set_embedded_opmask_register_specifier(mask);
9156 if (merge) {
9157 attributes.reset_is_clear_context();
9158 }
9159 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9160 emit_int8(0x40);
9161 emit_operand(dst, src, 0);
9162 }
9163
9164 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9165 assert(VM_Version::supports_evex(), "");
9166 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9167 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9168 attributes.set_is_evex_instruction();
9169 attributes.set_embedded_opmask_register_specifier(mask);
9170 if (merge) {
9171 attributes.reset_is_clear_context();
9172 }
9173 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9174 emit_int16(0x59, (0xC0 | encode));
9175 }
9176
9177 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9178 InstructionMark im(this);
9179 assert(VM_Version::supports_evex(), "");
9180 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9181 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9182 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9183 attributes.set_is_evex_instruction();
9184 attributes.set_embedded_opmask_register_specifier(mask);
9185 if (merge) {
9186 attributes.reset_is_clear_context();
9187 }
9188 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9189 emit_int8(0x59);
9190 emit_operand(dst, src, 0);
9191 }
9192
9193 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9194 assert(VM_Version::supports_evex(), "");
9195 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9196 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9197 attributes.set_is_evex_instruction();
9198 attributes.set_embedded_opmask_register_specifier(mask);
9199 if (merge) {
9200 attributes.reset_is_clear_context();
9201 }
9202 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9203 emit_int16(0x59, (0xC0 | encode));
9204 }
9205
9206 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9207 InstructionMark im(this);
9208 assert(VM_Version::supports_evex(), "");
9209 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9210 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9211 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9212 attributes.set_is_evex_instruction();
9213 attributes.set_embedded_opmask_register_specifier(mask);
9214 if (merge) {
9215 attributes.reset_is_clear_context();
9216 }
9217 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9218 emit_int8(0x59);
9219 emit_operand(dst, src, 0);
9220 }
9221
9222 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9223 assert(VM_Version::supports_evex(), "");
9224 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9225 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9226 attributes.set_is_evex_instruction();
9227 attributes.set_embedded_opmask_register_specifier(mask);
9228 if (merge) {
9229 attributes.reset_is_clear_context();
9230 }
9231 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9232 emit_int16(0x51, (0xC0 | encode));
9233 }
9234
9235 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9236 InstructionMark im(this);
9237 assert(VM_Version::supports_evex(), "");
9238 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9239 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9240 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9241 attributes.set_is_evex_instruction();
9242 attributes.set_embedded_opmask_register_specifier(mask);
9243 if (merge) {
9244 attributes.reset_is_clear_context();
9245 }
9246 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9247 emit_int8(0x51);
9248 emit_operand(dst, src, 0);
9249 }
9250
9251 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9252 assert(VM_Version::supports_evex(), "");
9253 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9254 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9255 attributes.set_is_evex_instruction();
9256 attributes.set_embedded_opmask_register_specifier(mask);
9257 if (merge) {
9258 attributes.reset_is_clear_context();
9259 }
9260 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9261 emit_int16(0x51, (0xC0 | encode));
9262 }
9263
9264 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9265 InstructionMark im(this);
9266 assert(VM_Version::supports_evex(), "");
9267 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9268 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9269 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9270 attributes.set_is_evex_instruction();
9271 attributes.set_embedded_opmask_register_specifier(mask);
9272 if (merge) {
9273 attributes.reset_is_clear_context();
9274 }
9275 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9276 emit_int8(0x51);
9277 emit_operand(dst, src, 0);
9278 }
9279
9280
9281 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9282 assert(VM_Version::supports_evex(), "");
9283 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9284 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9285 attributes.set_is_evex_instruction();
9286 attributes.set_embedded_opmask_register_specifier(mask);
9287 if (merge) {
9288 attributes.reset_is_clear_context();
9289 }
9290 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9291 emit_int16(0x5E, (0xC0 | encode));
9292 }
9293
9294 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9295 InstructionMark im(this);
9296 assert(VM_Version::supports_evex(), "");
9297 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9298 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9299 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9300 attributes.set_is_evex_instruction();
9301 attributes.set_embedded_opmask_register_specifier(mask);
9302 if (merge) {
9303 attributes.reset_is_clear_context();
9304 }
9305 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9306 emit_int8(0x5E);
9307 emit_operand(dst, src, 0);
9308 }
9309
9310 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9311 assert(VM_Version::supports_evex(), "");
9312 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9313 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9314 attributes.set_is_evex_instruction();
9315 attributes.set_embedded_opmask_register_specifier(mask);
9316 if (merge) {
9317 attributes.reset_is_clear_context();
9318 }
9319 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9320 emit_int16(0x5E, (0xC0 | encode));
9321 }
9322
9323 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9324 InstructionMark im(this);
9325 assert(VM_Version::supports_evex(), "");
9326 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9327 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9328 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9329 attributes.set_is_evex_instruction();
9330 attributes.set_embedded_opmask_register_specifier(mask);
9331 if (merge) {
9332 attributes.reset_is_clear_context();
9333 }
9334 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9335 emit_int8(0x5E);
9336 emit_operand(dst, src, 0);
9337 }
9338
9339 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9340 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9341 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9342 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9343 attributes.set_is_evex_instruction();
9344 attributes.set_embedded_opmask_register_specifier(mask);
9345 if (merge) {
9346 attributes.reset_is_clear_context();
9347 }
9348 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9349 emit_int16(0x1C, (0xC0 | encode));
9350 }
9351
9352
9353 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9354 InstructionMark im(this);
9355 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9356 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9357 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9358 attributes.set_is_evex_instruction();
9359 attributes.set_embedded_opmask_register_specifier(mask);
9360 if (merge) {
9361 attributes.reset_is_clear_context();
9362 }
9363 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9364 emit_int8(0x1C);
9365 emit_operand(dst, src, 0);
9366 }
9367
9368 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9369 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9370 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9371 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9372 attributes.set_is_evex_instruction();
9373 attributes.set_embedded_opmask_register_specifier(mask);
9374 if (merge) {
9375 attributes.reset_is_clear_context();
9376 }
9377 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9378 emit_int16(0x1D, (0xC0 | encode));
9379 }
9380
9381
9382 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9383 InstructionMark im(this);
9384 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9385 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9386 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9387 attributes.set_is_evex_instruction();
9388 attributes.set_embedded_opmask_register_specifier(mask);
9389 if (merge) {
9390 attributes.reset_is_clear_context();
9391 }
9392 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9393 emit_int8(0x1D);
9394 emit_operand(dst, src, 0);
9395 }
9396
9397 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9398 assert(VM_Version::supports_evex(), "");
9399 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9400 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9401 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9402 attributes.set_is_evex_instruction();
9403 attributes.set_embedded_opmask_register_specifier(mask);
9404 if (merge) {
9405 attributes.reset_is_clear_context();
9406 }
9407 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9408 emit_int16(0x1E, (0xC0 | encode));
9409 }
9410
9411
9412 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9413 InstructionMark im(this);
9414 assert(VM_Version::supports_evex(), "");
9415 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9416 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9417 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9418 attributes.set_is_evex_instruction();
9419 attributes.set_embedded_opmask_register_specifier(mask);
9420 if (merge) {
9421 attributes.reset_is_clear_context();
9422 }
9423 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9424 emit_int8(0x1E);
9425 emit_operand(dst, src, 0);
9426 }
9427
9428 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9429 assert(VM_Version::supports_evex(), "");
9430 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9431 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9432 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9433 attributes.set_is_evex_instruction();
9434 attributes.set_embedded_opmask_register_specifier(mask);
9435 if (merge) {
9436 attributes.reset_is_clear_context();
9437 }
9438 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9439 emit_int16(0x1F, (0xC0 | encode));
9440 }
9441
9442
9443 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9444 InstructionMark im(this);
9445 assert(VM_Version::supports_evex(), "");
9446 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9447 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9448 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9449 attributes.set_is_evex_instruction();
9450 attributes.set_embedded_opmask_register_specifier(mask);
9451 if (merge) {
9452 attributes.reset_is_clear_context();
9453 }
9454 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9455 emit_int8(0x1F);
9456 emit_operand(dst, src, 0);
9457 }
9458
9459 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9460 assert(VM_Version::supports_evex(), "");
9461 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9462 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9463 attributes.set_is_evex_instruction();
9464 attributes.set_embedded_opmask_register_specifier(mask);
9465 if (merge) {
9466 attributes.reset_is_clear_context();
9467 }
9468 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9469 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9470 }
9471
9472 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9473 InstructionMark im(this);
9474 assert(VM_Version::supports_evex(), "");
9475 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9476 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9477 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9478 attributes.set_is_evex_instruction();
9479 attributes.set_embedded_opmask_register_specifier(mask);
9480 if (merge) {
9481 attributes.reset_is_clear_context();
9482 }
9483 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9484 emit_int8((unsigned char)0xA8);
9485 emit_operand(dst, src, 0);
9486 }
9487
9488 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9489 assert(VM_Version::supports_evex(), "");
9490 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9491 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9492 attributes.set_is_evex_instruction();
9493 attributes.set_embedded_opmask_register_specifier(mask);
9494 if (merge) {
9495 attributes.reset_is_clear_context();
9496 }
9497 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9498 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9499 }
9500
9501 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9502 InstructionMark im(this);
9503 assert(VM_Version::supports_evex(), "");
9504 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9505 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9506 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9507 attributes.set_is_evex_instruction();
9508 attributes.set_embedded_opmask_register_specifier(mask);
9509 if (merge) {
9510 attributes.reset_is_clear_context();
9511 }
9512 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9513 emit_int8((unsigned char)0xA8);
9514 emit_operand(dst, src, 0);
9515 }
9516
9517 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9518 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9519 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9520 attributes.set_is_evex_instruction();
9521 attributes.set_embedded_opmask_register_specifier(mask);
9522 if (merge) {
9523 attributes.reset_is_clear_context();
9524 }
9525 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9526 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9527 }
9528
9529 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9530 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9531 InstructionMark im(this);
9532 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9533 attributes.set_is_evex_instruction();
9534 attributes.set_embedded_opmask_register_specifier(mask);
9535 if (merge) {
9536 attributes.reset_is_clear_context();
9537 }
9538 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9539 emit_int8((unsigned char)0x8D);
9540 emit_operand(dst, src, 0);
9541 }
9542
9543 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9544 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9545 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9546 attributes.set_is_evex_instruction();
9547 attributes.set_embedded_opmask_register_specifier(mask);
9548 if (merge) {
9549 attributes.reset_is_clear_context();
9550 }
9551 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9552 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9553 }
9554
9555 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9556 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9557 InstructionMark im(this);
9558 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9559 attributes.set_is_evex_instruction();
9560 attributes.set_embedded_opmask_register_specifier(mask);
9561 if (merge) {
9562 attributes.reset_is_clear_context();
9563 }
9564 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9565 emit_int8((unsigned char)0x8D);
9566 emit_operand(dst, src, 0);
9567 }
9568
9569 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9570 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9571 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9572 attributes.set_is_evex_instruction();
9573 attributes.set_embedded_opmask_register_specifier(mask);
9574 if (merge) {
9575 attributes.reset_is_clear_context();
9576 }
9577 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9578 emit_int16(0x36, (0xC0 | encode));
9579 }
9580
9581 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9582 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9583 InstructionMark im(this);
9584 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9585 attributes.set_is_evex_instruction();
9586 attributes.set_embedded_opmask_register_specifier(mask);
9587 if (merge) {
9588 attributes.reset_is_clear_context();
9589 }
9590 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9591 emit_int8(0x36);
9592 emit_operand(dst, src, 0);
9593 }
9594
9595 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9596 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9597 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9598 attributes.set_is_evex_instruction();
9599 attributes.set_embedded_opmask_register_specifier(mask);
9600 if (merge) {
9601 attributes.reset_is_clear_context();
9602 }
9603 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9604 emit_int16(0x36, (0xC0 | encode));
9605 }
9606
9607 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9608 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9609 InstructionMark im(this);
9610 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9611 attributes.set_is_evex_instruction();
9612 attributes.set_embedded_opmask_register_specifier(mask);
9613 if (merge) {
9614 attributes.reset_is_clear_context();
9615 }
9616 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9617 emit_int8(0x36);
9618 emit_operand(dst, src, 0);
9619 }
9620
9621 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9622 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9623 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9624 attributes.set_is_evex_instruction();
9625 attributes.set_embedded_opmask_register_specifier(mask);
9626 if (merge) {
9627 attributes.reset_is_clear_context();
9628 }
9629 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9630 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9631 }
9632
9633 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9634 assert(VM_Version::supports_evex(), "");
9635 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9636 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9637 attributes.set_is_evex_instruction();
9638 attributes.set_embedded_opmask_register_specifier(mask);
9639 if (merge) {
9640 attributes.reset_is_clear_context();
9641 }
9642 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9643 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9644 }
9645
9646 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9647 assert(VM_Version::supports_evex(), "");
9648 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9649 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9650 attributes.set_is_evex_instruction();
9651 attributes.set_embedded_opmask_register_specifier(mask);
9652 if (merge) {
9653 attributes.reset_is_clear_context();
9654 }
9655 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9656 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9657 }
9658
9659 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9660 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9661 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9662 attributes.set_is_evex_instruction();
9663 attributes.set_embedded_opmask_register_specifier(mask);
9664 if (merge) {
9665 attributes.reset_is_clear_context();
9666 }
9667 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9668 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9669 }
9670
9671 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9672 assert(VM_Version::supports_evex(), "");
9673 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9674 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9675 attributes.set_is_evex_instruction();
9676 attributes.set_embedded_opmask_register_specifier(mask);
9677 if (merge) {
9678 attributes.reset_is_clear_context();
9679 }
9680 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9681 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9682 }
9683
9684 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9685 assert(VM_Version::supports_evex(), "");
9686 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9687 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9688 attributes.set_is_evex_instruction();
9689 attributes.set_embedded_opmask_register_specifier(mask);
9690 if (merge) {
9691 attributes.reset_is_clear_context();
9692 }
9693 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9694 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9695 }
9696
9697 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9698 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9699 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9700 attributes.set_is_evex_instruction();
9701 attributes.set_embedded_opmask_register_specifier(mask);
9702 if (merge) {
9703 attributes.reset_is_clear_context();
9704 }
9705 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9706 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9707 }
9708
9709 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9710 assert(VM_Version::supports_evex(), "");
9711 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9712 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9713 attributes.set_is_evex_instruction();
9714 attributes.set_embedded_opmask_register_specifier(mask);
9715 if (merge) {
9716 attributes.reset_is_clear_context();
9717 }
9718 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9719 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9720 }
9721
9722 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9723 assert(VM_Version::supports_evex(), "");
9724 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9725 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9726 attributes.set_is_evex_instruction();
9727 attributes.set_embedded_opmask_register_specifier(mask);
9728 if (merge) {
9729 attributes.reset_is_clear_context();
9730 }
9731 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9732 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9733 }
9734
9735 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9736 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9737 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9738 attributes.set_is_evex_instruction();
9739 attributes.set_embedded_opmask_register_specifier(mask);
9740 if (merge) {
9741 attributes.reset_is_clear_context();
9742 }
9743 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9744 emit_int16((unsigned char)0xF1, (0xC0 | encode));
9745 }
9746
9747 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9748 assert(VM_Version::supports_evex(), "");
9749 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9750 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9751 attributes.set_is_evex_instruction();
9752 attributes.set_embedded_opmask_register_specifier(mask);
9753 if (merge) {
9754 attributes.reset_is_clear_context();
9755 }
9756 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9757 emit_int16((unsigned char)0xF2, (0xC0 | encode));
9758 }
9759
9760 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9761 assert(VM_Version::supports_evex(), "");
9762 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9763 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9764 attributes.set_is_evex_instruction();
9765 attributes.set_embedded_opmask_register_specifier(mask);
9766 if (merge) {
9767 attributes.reset_is_clear_context();
9768 }
9769 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9770 emit_int16((unsigned char)0xF3, (0xC0 | encode));
9771 }
9772
9773 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9774 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9775 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9776 attributes.set_is_evex_instruction();
9777 attributes.set_embedded_opmask_register_specifier(mask);
9778 if (merge) {
9779 attributes.reset_is_clear_context();
9780 }
9781 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9782 emit_int16((unsigned char)0xD1, (0xC0 | encode));
9783 }
9784
9785 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9786 assert(VM_Version::supports_evex(), "");
9787 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9788 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9789 attributes.set_is_evex_instruction();
9790 attributes.set_embedded_opmask_register_specifier(mask);
9791 if (merge) {
9792 attributes.reset_is_clear_context();
9793 }
9794 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9795 emit_int16((unsigned char)0xD2, (0xC0 | encode));
9796 }
9797
9798 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9799 assert(VM_Version::supports_evex(), "");
9800 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9801 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9802 attributes.set_is_evex_instruction();
9803 attributes.set_embedded_opmask_register_specifier(mask);
9804 if (merge) {
9805 attributes.reset_is_clear_context();
9806 }
9807 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9808 emit_int16((unsigned char)0xD3, (0xC0 | encode));
9809 }
9810
9811 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9812 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9813 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9814 attributes.set_is_evex_instruction();
9815 attributes.set_embedded_opmask_register_specifier(mask);
9816 if (merge) {
9817 attributes.reset_is_clear_context();
9818 }
9819 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9820 emit_int16((unsigned char)0xE1, (0xC0 | encode));
9821 }
9822
9823 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9824 assert(VM_Version::supports_evex(), "");
9825 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9826 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9827 attributes.set_is_evex_instruction();
9828 attributes.set_embedded_opmask_register_specifier(mask);
9829 if (merge) {
9830 attributes.reset_is_clear_context();
9831 }
9832 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9833 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9834 }
9835
9836 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9837 assert(VM_Version::supports_evex(), "");
9838 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9839 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9840 attributes.set_is_evex_instruction();
9841 attributes.set_embedded_opmask_register_specifier(mask);
9842 if (merge) {
9843 attributes.reset_is_clear_context();
9844 }
9845 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9846 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9847 }
9848
9849 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9850 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9851 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9852 attributes.set_is_evex_instruction();
9853 attributes.set_embedded_opmask_register_specifier(mask);
9854 if (merge) {
9855 attributes.reset_is_clear_context();
9856 }
9857 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9858 emit_int16(0x12, (0xC0 | encode));
9859 }
9860
9861 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9862 assert(VM_Version::supports_evex(), "");
9863 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9864 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9865 attributes.set_is_evex_instruction();
9866 attributes.set_embedded_opmask_register_specifier(mask);
9867 if (merge) {
9868 attributes.reset_is_clear_context();
9869 }
9870 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9871 emit_int16(0x47, (0xC0 | encode));
9872 }
9873
9874 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9875 assert(VM_Version::supports_evex(), "");
9876 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9877 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9878 attributes.set_is_evex_instruction();
9879 attributes.set_embedded_opmask_register_specifier(mask);
9880 if (merge) {
9881 attributes.reset_is_clear_context();
9882 }
9883 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9884 emit_int16(0x47, (0xC0 | encode));
9885 }
9886
9887 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9888 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9889 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9890 attributes.set_is_evex_instruction();
9891 attributes.set_embedded_opmask_register_specifier(mask);
9892 if (merge) {
9893 attributes.reset_is_clear_context();
9894 }
9895 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9896 emit_int16(0x10, (0xC0 | encode));
9897 }
9898
9899 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9900 assert(VM_Version::supports_evex(), "");
9901 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9902 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9903 attributes.set_is_evex_instruction();
9904 attributes.set_embedded_opmask_register_specifier(mask);
9905 if (merge) {
9906 attributes.reset_is_clear_context();
9907 }
9908 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9909 emit_int16(0x45, (0xC0 | encode));
9910 }
9911
9912 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9913 assert(VM_Version::supports_evex(), "");
9914 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9915 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9916 attributes.set_is_evex_instruction();
9917 attributes.set_embedded_opmask_register_specifier(mask);
9918 if (merge) {
9919 attributes.reset_is_clear_context();
9920 }
9921 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9922 emit_int16(0x45, (0xC0 | encode));
9923 }
9924
9925 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9926 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9927 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9928 attributes.set_is_evex_instruction();
9929 attributes.set_embedded_opmask_register_specifier(mask);
9930 if (merge) {
9931 attributes.reset_is_clear_context();
9932 }
9933 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9934 emit_int16(0x11, (0xC0 | encode));
9935 }
9936
9937 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9938 assert(VM_Version::supports_evex(), "");
9939 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9940 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9941 attributes.set_is_evex_instruction();
9942 attributes.set_embedded_opmask_register_specifier(mask);
9943 if (merge) {
9944 attributes.reset_is_clear_context();
9945 }
9946 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9947 emit_int16(0x46, (0xC0 | encode));
9948 }
9949
9950 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9951 assert(VM_Version::supports_evex(), "");
9952 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9953 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9954 attributes.set_is_evex_instruction();
9955 attributes.set_embedded_opmask_register_specifier(mask);
9956 if (merge) {
9957 attributes.reset_is_clear_context();
9958 }
9959 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9960 emit_int16(0x46, (0xC0 | encode));
9961 }
9962
9963 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9964 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9965 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9966 attributes.set_is_evex_instruction();
9967 attributes.set_embedded_opmask_register_specifier(mask);
9968 if (merge) {
9969 attributes.reset_is_clear_context();
9970 }
9971 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9972 emit_int16(0x38, (0xC0 | encode));
9973 }
9974
9975 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9976 assert(VM_Version::supports_avx512bw(), "");
9977 InstructionMark im(this);
9978 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9979 attributes.set_is_evex_instruction();
9980 attributes.set_embedded_opmask_register_specifier(mask);
9981 if (merge) {
9982 attributes.reset_is_clear_context();
9983 }
9984 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9985 emit_int8(0x38);
9986 emit_operand(dst, src, 0);
9987 }
9988
9989 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9990 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9991 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9992 attributes.set_is_evex_instruction();
9993 attributes.set_embedded_opmask_register_specifier(mask);
9994 if (merge) {
9995 attributes.reset_is_clear_context();
9996 }
9997 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9998 emit_int16((unsigned char)0xEA, (0xC0 | encode));
9999 }
10000
10001 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10002 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10003 InstructionMark im(this);
10004 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10005 attributes.set_is_evex_instruction();
10006 attributes.set_embedded_opmask_register_specifier(mask);
10007 if (merge) {
10008 attributes.reset_is_clear_context();
10009 }
10010 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10011 emit_int8((unsigned char)0xEA);
10012 emit_operand(dst, src, 0);
10013 }
10014
10015 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10016 assert(VM_Version::supports_evex(), "");
10017 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10018 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10019 attributes.set_is_evex_instruction();
10020 attributes.set_embedded_opmask_register_specifier(mask);
10021 if (merge) {
10022 attributes.reset_is_clear_context();
10023 }
10024 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10025 emit_int16(0x39, (0xC0 | encode));
10026 }
10027
10028 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10029 assert(VM_Version::supports_evex(), "");
10030 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10031 InstructionMark im(this);
10032 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10033 attributes.set_is_evex_instruction();
10034 attributes.set_embedded_opmask_register_specifier(mask);
10035 if (merge) {
10036 attributes.reset_is_clear_context();
10037 }
10038 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10039 emit_int8(0x39);
10040 emit_operand(dst, src, 0);
10041 }
10042
10043 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10044 assert(VM_Version::supports_evex(), "");
10045 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10046 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10047 attributes.set_is_evex_instruction();
10048 attributes.set_embedded_opmask_register_specifier(mask);
10049 if (merge) {
10050 attributes.reset_is_clear_context();
10051 }
10052 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10053 emit_int16(0x39, (0xC0 | encode));
10054 }
10055
10056 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10057 assert(VM_Version::supports_evex(), "");
10058 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10059 InstructionMark im(this);
10060 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10061 attributes.set_is_evex_instruction();
10062 attributes.set_embedded_opmask_register_specifier(mask);
10063 if (merge) {
10064 attributes.reset_is_clear_context();
10065 }
10066 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10067 emit_int8(0x39);
10068 emit_operand(dst, src, 0);
10069 }
10070
10071
10072 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10073 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10074 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10075 attributes.set_is_evex_instruction();
10076 attributes.set_embedded_opmask_register_specifier(mask);
10077 if (merge) {
10078 attributes.reset_is_clear_context();
10079 }
10080 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10081 emit_int16(0x3C, (0xC0 | encode));
10082 }
10083
10084 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10085 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10086 InstructionMark im(this);
10087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10088 attributes.set_is_evex_instruction();
10089 attributes.set_embedded_opmask_register_specifier(mask);
10090 if (merge) {
10091 attributes.reset_is_clear_context();
10092 }
10093 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10094 emit_int8(0x3C);
10095 emit_operand(dst, src, 0);
10096 }
10097
10098 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10099 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10100 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10101 attributes.set_is_evex_instruction();
10102 attributes.set_embedded_opmask_register_specifier(mask);
10103 if (merge) {
10104 attributes.reset_is_clear_context();
10105 }
10106 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10107 emit_int16((unsigned char)0xEE, (0xC0 | encode));
10108 }
10109
10110 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10111 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10112 InstructionMark im(this);
10113 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10114 attributes.set_is_evex_instruction();
10115 attributes.set_embedded_opmask_register_specifier(mask);
10116 if (merge) {
10117 attributes.reset_is_clear_context();
10118 }
10119 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10120 emit_int8((unsigned char)0xEE);
10121 emit_operand(dst, src, 0);
10122 }
10123
10124 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10125 assert(VM_Version::supports_evex(), "");
10126 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10127 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10128 attributes.set_is_evex_instruction();
10129 attributes.set_embedded_opmask_register_specifier(mask);
10130 if (merge) {
10131 attributes.reset_is_clear_context();
10132 }
10133 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10134 emit_int16(0x3D, (0xC0 | encode));
10135 }
10136
10137 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10138 assert(VM_Version::supports_evex(), "");
10139 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10140 InstructionMark im(this);
10141 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10142 attributes.set_is_evex_instruction();
10143 attributes.set_embedded_opmask_register_specifier(mask);
10144 if (merge) {
10145 attributes.reset_is_clear_context();
10146 }
10147 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10148 emit_int8(0x3D);
10149 emit_operand(dst, src, 0);
10150 }
10151
10152 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10153 assert(VM_Version::supports_evex(), "");
10154 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10155 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10156 attributes.set_is_evex_instruction();
10157 attributes.set_embedded_opmask_register_specifier(mask);
10158 if (merge) {
10159 attributes.reset_is_clear_context();
10160 }
10161 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10162 emit_int16(0x3D, (0xC0 | encode));
10163 }
10164
10165 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10166 assert(VM_Version::supports_evex(), "");
10167 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10168 InstructionMark im(this);
10169 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10170 attributes.set_is_evex_instruction();
10171 attributes.set_embedded_opmask_register_specifier(mask);
10172 if (merge) {
10173 attributes.reset_is_clear_context();
10174 }
10175 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10176 emit_int8(0x3D);
10177 emit_operand(dst, src, 0);
10178 }
10179
10180 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10181 assert(VM_Version::supports_evex(), "requires EVEX support");
10182 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10183 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10184 attributes.set_is_evex_instruction();
10185 attributes.set_embedded_opmask_register_specifier(mask);
10186 if (merge) {
10187 attributes.reset_is_clear_context();
10188 }
10189 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10190 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10191 }
10192
10193 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10194 assert(VM_Version::supports_evex(), "requires EVEX support");
10195 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10196 assert(dst != xnoreg, "sanity");
10197 InstructionMark im(this);
10198 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10199 attributes.set_is_evex_instruction();
10200 attributes.set_embedded_opmask_register_specifier(mask);
10201 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10202 if (merge) {
10203 attributes.reset_is_clear_context();
10204 }
10205 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10206 emit_int8(0x25);
10207 emit_operand(dst, src3, 1);
10208 emit_int8(imm8);
10209 }
10210
10211 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10212 assert(VM_Version::supports_evex(), "requires EVEX support");
10213 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10214 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10215 attributes.set_is_evex_instruction();
10216 attributes.set_embedded_opmask_register_specifier(mask);
10217 if (merge) {
10218 attributes.reset_is_clear_context();
10219 }
10220 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10221 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10222 }
10223
10224 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10225 assert(VM_Version::supports_evex(), "requires EVEX support");
10226 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10227 assert(dst != xnoreg, "sanity");
10228 InstructionMark im(this);
10229 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10230 attributes.set_is_evex_instruction();
10231 attributes.set_embedded_opmask_register_specifier(mask);
10232 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10233 if (merge) {
10234 attributes.reset_is_clear_context();
10235 }
10236 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10237 emit_int8(0x25);
10238 emit_operand(dst, src3, 1);
10239 emit_int8(imm8);
10240 }
10241
10242 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10243 assert(VM_Version::supports_gfni(), "");
10244 assert(VM_Version::supports_sse(), "");
10245 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10246 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10247 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10248 }
10249
10250 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10251 assert(VM_Version::supports_gfni(), "requires GFNI support");
10252 assert(VM_Version::supports_sse(), "");
10253 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10254 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10255 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10256 }
10257
10258 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10259 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10260 assert(UseAVX >= 2, "");
10261 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10262 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10263 emit_int16(0x58, (0xC0 | encode));
10264 }
10265
10266 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10267 assert(VM_Version::supports_avx2(), "");
10268 assert(dst != xnoreg, "sanity");
10269 InstructionMark im(this);
10270 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10271 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10272 // swap src<->dst for encoding
10273 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10274 emit_int8(0x58);
10275 emit_operand(dst, src, 0);
10276 }
10277
10278 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10279 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10280 assert(VM_Version::supports_avx2(), "");
10281 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10282 attributes.set_rex_vex_w_reverted();
10283 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10284 emit_int16(0x59, (0xC0 | encode));
10285 }
10286
10287 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10288 assert(VM_Version::supports_avx2(), "");
10289 assert(dst != xnoreg, "sanity");
10290 InstructionMark im(this);
10291 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10292 attributes.set_rex_vex_w_reverted();
10293 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10294 // swap src<->dst for encoding
10295 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10296 emit_int8(0x59);
10297 emit_operand(dst, src, 0);
10298 }
10299
10300 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10301 assert(vector_len != Assembler::AVX_128bit, "");
10302 assert(VM_Version::supports_evex(), "");
10303 assert(dst != xnoreg, "sanity");
10304 InstructionMark im(this);
10305 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10306 attributes.set_rex_vex_w_reverted();
10307 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10308 // swap src<->dst for encoding
10309 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10310 emit_int8(0x5A);
10311 emit_operand(dst, src, 0);
10312 }
10313
10314 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10315 assert(vector_len != Assembler::AVX_128bit, "");
10316 assert(VM_Version::supports_avx512dq(), "");
10317 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10318 attributes.set_rex_vex_w_reverted();
10319 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10320 emit_int16(0x5A, (0xC0 | encode));
10321 }
10322
10323 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10324 assert(vector_len != Assembler::AVX_128bit, "");
10325 assert(VM_Version::supports_avx512dq(), "");
10326 assert(dst != xnoreg, "sanity");
10327 InstructionMark im(this);
10328 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10329 attributes.set_rex_vex_w_reverted();
10330 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10331 // swap src<->dst for encoding
10332 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10333 emit_int8(0x5A);
10334 emit_operand(dst, src, 0);
10335 }
10336
10337 // scalar single/double precision replicate
10338
10339 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10340 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10341 assert(VM_Version::supports_avx2(), "");
10342 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10343 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10344 emit_int16(0x18, (0xC0 | encode));
10345 }
10346
10347 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10348 assert(VM_Version::supports_avx(), "");
10349 assert(dst != xnoreg, "sanity");
10350 InstructionMark im(this);
10351 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10352 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10353 // swap src<->dst for encoding
10354 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10355 emit_int8(0x18);
10356 emit_operand(dst, src, 0);
10357 }
10358
10359 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10360 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10361 assert(VM_Version::supports_avx2(), "");
10362 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10363 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10364 attributes.set_rex_vex_w_reverted();
10365 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10366 emit_int16(0x19, (0xC0 | encode));
10367 }
10368
10369 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10370 assert(VM_Version::supports_avx(), "");
10371 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10372 assert(dst != xnoreg, "sanity");
10373 InstructionMark im(this);
10374 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10375 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10376 attributes.set_rex_vex_w_reverted();
10377 // swap src<->dst for encoding
10378 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10379 emit_int8(0x19);
10380 emit_operand(dst, src, 0);
10381 }
10382
10383 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10384 assert(VM_Version::supports_avx(), "");
10385 assert(vector_len == AVX_256bit, "");
10386 assert(dst != xnoreg, "sanity");
10387 InstructionMark im(this);
10388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10389 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10390 // swap src<->dst for encoding
10391 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10392 emit_int8(0x1A);
10393 emit_operand(dst, src, 0);
10394 }
10395
10396 // gpr source broadcast forms
10397
10398 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10399 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10400 assert(VM_Version::supports_avx512bw(), "");
10401 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10402 attributes.set_is_evex_instruction();
10403 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10404 emit_int16(0x7A, (0xC0 | encode));
10405 }
10406
10407 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10408 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10409 assert(VM_Version::supports_avx512bw(), "");
10410 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10411 attributes.set_is_evex_instruction();
10412 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10413 emit_int16(0x7B, (0xC0 | encode));
10414 }
10415
10416 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10417 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10418 assert(VM_Version::supports_evex(), "");
10419 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10420 attributes.set_is_evex_instruction();
10421 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10422 emit_int16(0x7C, (0xC0 | encode));
10423 }
10424
10425 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10426 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10427 assert(VM_Version::supports_evex(), "");
10428 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10429 attributes.set_is_evex_instruction();
10430 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10431 emit_int16(0x7C, (0xC0 | encode));
10432 }
10433
10434 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10435 assert(VM_Version::supports_avx2(), "");
10436 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10437 assert(dst != xnoreg, "sanity");
10438 assert(src.isxmmindex(),"expected to be xmm index");
10439 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10440 InstructionMark im(this);
10441 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10442 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10443 emit_int8((unsigned char)0x90);
10444 emit_operand(dst, src, 0);
10445 }
10446
10447 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10448 assert(VM_Version::supports_avx2(), "");
10449 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10450 assert(dst != xnoreg, "sanity");
10451 assert(src.isxmmindex(),"expected to be xmm index");
10452 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10453 InstructionMark im(this);
10454 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10455 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10456 emit_int8((unsigned char)0x90);
10457 emit_operand(dst, src, 0);
10458 }
10459
10460 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10461 assert(VM_Version::supports_avx2(), "");
10462 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10463 assert(dst != xnoreg, "sanity");
10464 assert(src.isxmmindex(),"expected to be xmm index");
10465 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10466 InstructionMark im(this);
10467 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10468 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10469 emit_int8((unsigned char)0x92);
10470 emit_operand(dst, src, 0);
10471 }
10472
10473 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10474 assert(VM_Version::supports_avx2(), "");
10475 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10476 assert(dst != xnoreg, "sanity");
10477 assert(src.isxmmindex(),"expected to be xmm index");
10478 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10479 InstructionMark im(this);
10480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10481 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10482 emit_int8((unsigned char)0x92);
10483 emit_operand(dst, src, 0);
10484 }
10485 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10486 assert(VM_Version::supports_evex(), "");
10487 assert(dst != xnoreg, "sanity");
10488 assert(src.isxmmindex(),"expected to be xmm index");
10489 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10490 assert(mask != k0, "instruction will #UD if mask is in k0");
10491 InstructionMark im(this);
10492 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10493 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10494 attributes.reset_is_clear_context();
10495 attributes.set_embedded_opmask_register_specifier(mask);
10496 attributes.set_is_evex_instruction();
10497 // swap src<->dst for encoding
10498 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10499 emit_int8((unsigned char)0x90);
10500 emit_operand(dst, src, 0);
10501 }
10502
10503 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10504 assert(VM_Version::supports_evex(), "");
10505 assert(dst != xnoreg, "sanity");
10506 assert(src.isxmmindex(),"expected to be xmm index");
10507 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10508 assert(mask != k0, "instruction will #UD if mask is in k0");
10509 InstructionMark im(this);
10510 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10511 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10512 attributes.reset_is_clear_context();
10513 attributes.set_embedded_opmask_register_specifier(mask);
10514 attributes.set_is_evex_instruction();
10515 // swap src<->dst for encoding
10516 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10517 emit_int8((unsigned char)0x90);
10518 emit_operand(dst, src, 0);
10519 }
10520
10521 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10522 assert(VM_Version::supports_evex(), "");
10523 assert(dst != xnoreg, "sanity");
10524 assert(src.isxmmindex(),"expected to be xmm index");
10525 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10526 assert(mask != k0, "instruction will #UD if mask is in k0");
10527 InstructionMark im(this);
10528 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10529 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10530 attributes.reset_is_clear_context();
10531 attributes.set_embedded_opmask_register_specifier(mask);
10532 attributes.set_is_evex_instruction();
10533 // swap src<->dst for encoding
10534 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10535 emit_int8((unsigned char)0x92);
10536 emit_operand(dst, src, 0);
10537 }
10538
10539 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10540 assert(VM_Version::supports_evex(), "");
10541 assert(dst != xnoreg, "sanity");
10542 assert(src.isxmmindex(),"expected to be xmm index");
10543 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10544 assert(mask != k0, "instruction will #UD if mask is in k0");
10545 InstructionMark im(this);
10546 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10547 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10548 attributes.reset_is_clear_context();
10549 attributes.set_embedded_opmask_register_specifier(mask);
10550 attributes.set_is_evex_instruction();
10551 // swap src<->dst for encoding
10552 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10553 emit_int8((unsigned char)0x92);
10554 emit_operand(dst, src, 0);
10555 }
10556
10557 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10558 assert(VM_Version::supports_evex(), "");
10559 assert(mask != k0, "instruction will #UD if mask is in k0");
10560 InstructionMark im(this);
10561 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10562 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10563 attributes.reset_is_clear_context();
10564 attributes.set_embedded_opmask_register_specifier(mask);
10565 attributes.set_is_evex_instruction();
10566 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10567 emit_int8((unsigned char)0xA0);
10568 emit_operand(src, dst, 0);
10569 }
10570
10571 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10572 assert(VM_Version::supports_evex(), "");
10573 assert(mask != k0, "instruction will #UD if mask is in k0");
10574 InstructionMark im(this);
10575 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10576 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10577 attributes.reset_is_clear_context();
10578 attributes.set_embedded_opmask_register_specifier(mask);
10579 attributes.set_is_evex_instruction();
10580 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10581 emit_int8((unsigned char)0xA0);
10582 emit_operand(src, dst, 0);
10583 }
10584
10585 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10586 assert(VM_Version::supports_evex(), "");
10587 assert(mask != k0, "instruction will #UD if mask is in k0");
10588 InstructionMark im(this);
10589 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10590 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10591 attributes.reset_is_clear_context();
10592 attributes.set_embedded_opmask_register_specifier(mask);
10593 attributes.set_is_evex_instruction();
10594 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10595 emit_int8((unsigned char)0xA2);
10596 emit_operand(src, dst, 0);
10597 }
10598
10599 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10600 assert(VM_Version::supports_evex(), "");
10601 assert(mask != k0, "instruction will #UD if mask is in k0");
10602 InstructionMark im(this);
10603 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10604 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10605 attributes.reset_is_clear_context();
10606 attributes.set_embedded_opmask_register_specifier(mask);
10607 attributes.set_is_evex_instruction();
10608 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10609 emit_int8((unsigned char)0xA2);
10610 emit_operand(src, dst, 0);
10611 }
10612 // Carry-Less Multiplication Quadword
10613 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10614 assert(VM_Version::supports_clmul(), "");
10615 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10616 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10617 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10618 }
10619
10620 // Carry-Less Multiplication Quadword
10621 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10622 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10623 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10624 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10625 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10626 }
10627
10628 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10629 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10630 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10631 attributes.set_is_evex_instruction();
10632 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10633 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10634 }
10635
10636 void Assembler::vzeroupper_uncached() {
10637 if (VM_Version::supports_vzeroupper()) {
10638 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10639 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10640 emit_int8(0x77);
10641 }
10642 }
10643
10644 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10645 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10646 assert(VM_Version::supports_evex(), "");
10647 assert(VM_Version::supports_avx512dq(), "");
10648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10649 attributes.set_is_evex_instruction();
10650 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10651 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10652 }
10653
10654 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10655 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10656 assert(VM_Version::supports_evex(), "");
10657 assert(VM_Version::supports_avx512dq(), "");
10658 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10659 attributes.set_is_evex_instruction();
10660 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10661 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10662 }
10663
10664 void Assembler::fld_x(Address adr) {
10665 InstructionMark im(this);
10666 emit_int8((unsigned char)0xDB);
10667 emit_operand32(rbp, adr, 0);
10668 }
10669
10670 void Assembler::fstp_x(Address adr) {
10671 InstructionMark im(this);
10672 emit_int8((unsigned char)0xDB);
10673 emit_operand32(rdi, adr, 0);
10674 }
10675
10676 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10677 assert(reg->encoding() < 8, "no extended registers");
10678 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10679 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10680 }
10681
10682 #ifndef _LP64
10683 // 32bit only pieces of the assembler
10684
10685 void Assembler::emms() {
10686 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10687 emit_int16(0x0F, 0x77);
10688 }
10689
10690 void Assembler::vzeroupper() {
10691 vzeroupper_uncached();
10692 }
10693
10694 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10695 // NO PREFIX AS NEVER 64BIT
10696 InstructionMark im(this);
10697 emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10698 emit_data(imm32, rspec, 0);
10699 }
10700
10701 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10702 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10703 InstructionMark im(this);
10704 emit_int8((unsigned char)0x81);
10705 emit_operand(rdi, src1, 4);
10706 emit_data(imm32, rspec, 0);
10707 }
10708
10709 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10710 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10711 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
10712 void Assembler::cmpxchg8(Address adr) {
10713 InstructionMark im(this);
10714 emit_int16(0x0F, (unsigned char)0xC7);
10715 emit_operand(rcx, adr, 0);
10716 }
10717
10718 void Assembler::decl(Register dst) {
10719 // Don't use it directly. Use MacroAssembler::decrementl() instead.
10720 emit_int8(0x48 | dst->encoding());
10721 }
10722
10723 // 64bit doesn't use the x87
10724
10725 void Assembler::emit_farith(int b1, int b2, int i) {
10726 assert(isByte(b1) && isByte(b2), "wrong opcode");
10727 assert(0 <= i && i < 8, "illegal stack offset");
10728 emit_int16(b1, b2 + i);
10729 }
10730
10731 void Assembler::fabs() {
10732 emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10733 }
10734
10735 void Assembler::fadd(int i) {
10736 emit_farith(0xD8, 0xC0, i);
10737 }
10738
10739 void Assembler::fadd_d(Address src) {
10740 InstructionMark im(this);
10741 emit_int8((unsigned char)0xDC);
10742 emit_operand32(rax, src, 0);
10743 }
10744
10745 void Assembler::fadd_s(Address src) {
10746 InstructionMark im(this);
10747 emit_int8((unsigned char)0xD8);
10748 emit_operand32(rax, src, 0);
10749 }
10750
10751 void Assembler::fadda(int i) {
10752 emit_farith(0xDC, 0xC0, i);
10753 }
10754
10755 void Assembler::faddp(int i) {
10756 emit_farith(0xDE, 0xC0, i);
10757 }
10758
10759 void Assembler::fchs() {
10760 emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10761 }
10762
10763 void Assembler::fcom(int i) {
10764 emit_farith(0xD8, 0xD0, i);
10765 }
10766
10767 void Assembler::fcomp(int i) {
10768 emit_farith(0xD8, 0xD8, i);
10769 }
10770
10771 void Assembler::fcomp_d(Address src) {
10772 InstructionMark im(this);
10773 emit_int8((unsigned char)0xDC);
10774 emit_operand32(rbx, src, 0);
10775 }
10776
10777 void Assembler::fcomp_s(Address src) {
10778 InstructionMark im(this);
10779 emit_int8((unsigned char)0xD8);
10780 emit_operand32(rbx, src, 0);
10781 }
10782
10783 void Assembler::fcompp() {
10784 emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10785 }
10786
10787 void Assembler::fcos() {
10788 emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10789 }
10790
10791 void Assembler::fdecstp() {
10792 emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10793 }
10794
10795 void Assembler::fdiv(int i) {
10796 emit_farith(0xD8, 0xF0, i);
10797 }
10798
10799 void Assembler::fdiv_d(Address src) {
10800 InstructionMark im(this);
10801 emit_int8((unsigned char)0xDC);
10802 emit_operand32(rsi, src, 0);
10803 }
10804
10805 void Assembler::fdiv_s(Address src) {
10806 InstructionMark im(this);
10807 emit_int8((unsigned char)0xD8);
10808 emit_operand32(rsi, src, 0);
10809 }
10810
10811 void Assembler::fdiva(int i) {
10812 emit_farith(0xDC, 0xF8, i);
10813 }
10814
10815 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10816 // is erroneous for some of the floating-point instructions below.
10817
10818 void Assembler::fdivp(int i) {
10819 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10820 }
10821
10822 void Assembler::fdivr(int i) {
10823 emit_farith(0xD8, 0xF8, i);
10824 }
10825
10826 void Assembler::fdivr_d(Address src) {
10827 InstructionMark im(this);
10828 emit_int8((unsigned char)0xDC);
10829 emit_operand32(rdi, src, 0);
10830 }
10831
10832 void Assembler::fdivr_s(Address src) {
10833 InstructionMark im(this);
10834 emit_int8((unsigned char)0xD8);
10835 emit_operand32(rdi, src, 0);
10836 }
10837
10838 void Assembler::fdivra(int i) {
10839 emit_farith(0xDC, 0xF0, i);
10840 }
10841
10842 void Assembler::fdivrp(int i) {
10843 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10844 }
10845
10846 void Assembler::ffree(int i) {
10847 emit_farith(0xDD, 0xC0, i);
10848 }
10849
10850 void Assembler::fild_d(Address adr) {
10851 InstructionMark im(this);
10852 emit_int8((unsigned char)0xDF);
10853 emit_operand32(rbp, adr, 0);
10854 }
10855
10856 void Assembler::fild_s(Address adr) {
10857 InstructionMark im(this);
10858 emit_int8((unsigned char)0xDB);
10859 emit_operand32(rax, adr, 0);
10860 }
10861
10862 void Assembler::fincstp() {
10863 emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10864 }
10865
10866 void Assembler::finit() {
10867 emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10868 }
10869
10870 void Assembler::fist_s(Address adr) {
10871 InstructionMark im(this);
10872 emit_int8((unsigned char)0xDB);
10873 emit_operand32(rdx, adr, 0);
10874 }
10875
10876 void Assembler::fistp_d(Address adr) {
10877 InstructionMark im(this);
10878 emit_int8((unsigned char)0xDF);
10879 emit_operand32(rdi, adr, 0);
10880 }
10881
10882 void Assembler::fistp_s(Address adr) {
10883 InstructionMark im(this);
10884 emit_int8((unsigned char)0xDB);
10885 emit_operand32(rbx, adr, 0);
10886 }
10887
10888 void Assembler::fld1() {
10889 emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10890 }
10891
10892 void Assembler::fld_d(Address adr) {
10893 InstructionMark im(this);
10894 emit_int8((unsigned char)0xDD);
10895 emit_operand32(rax, adr, 0);
10896 }
10897
10898 void Assembler::fld_s(Address adr) {
10899 InstructionMark im(this);
10900 emit_int8((unsigned char)0xD9);
10901 emit_operand32(rax, adr, 0);
10902 }
10903
10904
10905 void Assembler::fld_s(int index) {
10906 emit_farith(0xD9, 0xC0, index);
10907 }
10908
10909 void Assembler::fldcw(Address src) {
10910 InstructionMark im(this);
10911 emit_int8((unsigned char)0xD9);
10912 emit_operand32(rbp, src, 0);
10913 }
10914
10915 void Assembler::fldenv(Address src) {
10916 InstructionMark im(this);
10917 emit_int8((unsigned char)0xD9);
10918 emit_operand32(rsp, src, 0);
10919 }
10920
10921 void Assembler::fldlg2() {
10922 emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10923 }
10924
10925 void Assembler::fldln2() {
10926 emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10927 }
10928
10929 void Assembler::fldz() {
10930 emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10931 }
10932
10933 void Assembler::flog() {
10934 fldln2();
10935 fxch();
10936 fyl2x();
10937 }
10938
10939 void Assembler::flog10() {
10940 fldlg2();
10941 fxch();
10942 fyl2x();
10943 }
10944
10945 void Assembler::fmul(int i) {
10946 emit_farith(0xD8, 0xC8, i);
10947 }
10948
10949 void Assembler::fmul_d(Address src) {
10950 InstructionMark im(this);
10951 emit_int8((unsigned char)0xDC);
10952 emit_operand32(rcx, src, 0);
10953 }
10954
10955 void Assembler::fmul_s(Address src) {
10956 InstructionMark im(this);
10957 emit_int8((unsigned char)0xD8);
10958 emit_operand32(rcx, src, 0);
10959 }
10960
10961 void Assembler::fmula(int i) {
10962 emit_farith(0xDC, 0xC8, i);
10963 }
10964
10965 void Assembler::fmulp(int i) {
10966 emit_farith(0xDE, 0xC8, i);
10967 }
10968
10969 void Assembler::fnsave(Address dst) {
10970 InstructionMark im(this);
10971 emit_int8((unsigned char)0xDD);
10972 emit_operand32(rsi, dst, 0);
10973 }
10974
10975 void Assembler::fnstcw(Address src) {
10976 InstructionMark im(this);
10977 emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10978 emit_operand32(rdi, src, 0);
10979 }
10980
10981 void Assembler::fnstsw_ax() {
10982 emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10983 }
10984
10985 void Assembler::fprem() {
10986 emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10987 }
10988
10989 void Assembler::fprem1() {
10990 emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10991 }
10992
10993 void Assembler::frstor(Address src) {
10994 InstructionMark im(this);
10995 emit_int8((unsigned char)0xDD);
10996 emit_operand32(rsp, src, 0);
10997 }
10998
10999 void Assembler::fsin() {
11000 emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
11001 }
11002
11003 void Assembler::fsqrt() {
11004 emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11005 }
11006
11007 void Assembler::fst_d(Address adr) {
11008 InstructionMark im(this);
11009 emit_int8((unsigned char)0xDD);
11010 emit_operand32(rdx, adr, 0);
11011 }
11012
11013 void Assembler::fst_s(Address adr) {
11014 InstructionMark im(this);
11015 emit_int8((unsigned char)0xD9);
11016 emit_operand32(rdx, adr, 0);
11017 }
11018
11019 void Assembler::fstp_d(Address adr) {
11020 InstructionMark im(this);
11021 emit_int8((unsigned char)0xDD);
11022 emit_operand32(rbx, adr, 0);
11023 }
11024
11025 void Assembler::fstp_d(int index) {
11026 emit_farith(0xDD, 0xD8, index);
11027 }
11028
11029 void Assembler::fstp_s(Address adr) {
11030 InstructionMark im(this);
11031 emit_int8((unsigned char)0xD9);
11032 emit_operand32(rbx, adr, 0);
11033 }
11034
11035 void Assembler::fsub(int i) {
11036 emit_farith(0xD8, 0xE0, i);
11037 }
11038
11039 void Assembler::fsub_d(Address src) {
11040 InstructionMark im(this);
11041 emit_int8((unsigned char)0xDC);
11042 emit_operand32(rsp, src, 0);
11043 }
11044
11045 void Assembler::fsub_s(Address src) {
11046 InstructionMark im(this);
11047 emit_int8((unsigned char)0xD8);
11048 emit_operand32(rsp, src, 0);
11049 }
11050
11051 void Assembler::fsuba(int i) {
11052 emit_farith(0xDC, 0xE8, i);
11053 }
11054
11055 void Assembler::fsubp(int i) {
11056 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11057 }
11058
11059 void Assembler::fsubr(int i) {
11060 emit_farith(0xD8, 0xE8, i);
11061 }
11062
11063 void Assembler::fsubr_d(Address src) {
11064 InstructionMark im(this);
11065 emit_int8((unsigned char)0xDC);
11066 emit_operand32(rbp, src, 0);
11067 }
11068
11069 void Assembler::fsubr_s(Address src) {
11070 InstructionMark im(this);
11071 emit_int8((unsigned char)0xD8);
11072 emit_operand32(rbp, src, 0);
11073 }
11074
11075 void Assembler::fsubra(int i) {
11076 emit_farith(0xDC, 0xE0, i);
11077 }
11078
11079 void Assembler::fsubrp(int i) {
11080 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11081 }
11082
11083 void Assembler::ftan() {
11084 emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11085 }
11086
11087 void Assembler::ftst() {
11088 emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11089 }
11090
11091 void Assembler::fucomi(int i) {
11092 // make sure the instruction is supported (introduced for P6, together with cmov)
11093 guarantee(VM_Version::supports_cmov(), "illegal instruction");
11094 emit_farith(0xDB, 0xE8, i);
11095 }
11096
11097 void Assembler::fucomip(int i) {
11098 // make sure the instruction is supported (introduced for P6, together with cmov)
11099 guarantee(VM_Version::supports_cmov(), "illegal instruction");
11100 emit_farith(0xDF, 0xE8, i);
11101 }
11102
11103 void Assembler::fwait() {
11104 emit_int8((unsigned char)0x9B);
11105 }
11106
11107 void Assembler::fxch(int i) {
11108 emit_farith(0xD9, 0xC8, i);
11109 }
11110
11111 void Assembler::fyl2x() {
11112 emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11113 }
11114
11115 void Assembler::frndint() {
11116 emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11117 }
11118
11119 void Assembler::f2xm1() {
11120 emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11121 }
11122
11123 void Assembler::fldl2e() {
11124 emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11125 }
11126 #endif // !_LP64
11127
11128 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11129 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11130 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11131 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
11132
11133 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11134 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11135 if (pre > 0) {
11136 emit_int8(simd_pre[pre]);
11137 }
11138 if (rex_w) {
11139 prefixq(adr, xreg);
11140 } else {
11141 prefix(adr, xreg);
11142 }
11143 if (opc > 0) {
11144 emit_int8(0x0F);
11145 int opc2 = simd_opc[opc];
11146 if (opc2 > 0) {
11147 emit_int8(opc2);
11148 }
11149 }
11150 }
11151
11152 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11153 if (pre > 0) {
11154 emit_int8(simd_pre[pre]);
11155 }
11156 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11157 if (opc > 0) {
11158 emit_int8(0x0F);
11159 int opc2 = simd_opc[opc];
11160 if (opc2 > 0) {
11161 emit_int8(opc2);
11162 }
11163 }
11164 return encode;
11165 }
11166
11167
11168 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11169 int vector_len = _attributes->get_vector_len();
11170 bool vex_w = _attributes->is_rex_vex_w();
11171 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11172 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11173 byte1 = (~byte1) & 0xE0;
11174 byte1 |= opc;
11175
11176 int byte2 = ((~nds_enc) & 0xf) << 3;
11177 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11178
11179 emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11180 } else {
11181 int byte1 = vex_r ? VEX_R : 0;
11182 byte1 = (~byte1) & 0x80;
11183 byte1 |= ((~nds_enc) & 0xf) << 3;
11184 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11185 emit_int16((unsigned char)VEX_2bytes, byte1);
11186 }
11187 }
11188
11189 // This is a 4 byte encoding
11190 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11191 // EVEX 0x62 prefix
11192 // byte1 = EVEX_4bytes;
11193
11194 bool vex_w = _attributes->is_rex_vex_w();
11195 int evex_encoding = (vex_w ? VEX_W : 0);
11196 // EVEX.b is not currently used for broadcast of single element or data rounding modes
11197 _attributes->set_evex_encoding(evex_encoding);
11198
11199 // P0: byte 2, initialized to RXBR`00mm
11200 // instead of not'd
11201 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11202 byte2 = (~byte2) & 0xF0;
11203 // confine opc opcode extensions in mm bits to lower two bits
11204 // of form {0F, 0F_38, 0F_3A}
11205 byte2 |= opc;
11206
11207 // P1: byte 3 as Wvvvv1pp
11208 int byte3 = ((~nds_enc) & 0xf) << 3;
11209 // p[10] is always 1
11210 byte3 |= EVEX_F;
11211 byte3 |= (vex_w & 1) << 7;
11212 // confine pre opcode extensions in pp bits to lower two bits
11213 // of form {66, F3, F2}
11214 byte3 |= pre;
11215
11216 // P2: byte 4 as zL'Lbv'aaa
11217 // kregs are implemented in the low 3 bits as aaa
11218 int byte4 = (_attributes->is_no_reg_mask()) ?
11219 0 :
11220 _attributes->get_embedded_opmask_register_specifier();
11221 // EVEX.v` for extending EVEX.vvvv or VIDX
11222 byte4 |= (evex_v ? 0: EVEX_V);
11223 // third EXEC.b for broadcast actions
11224 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11225 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11226 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11227 // last is EVEX.z for zero/merge actions
11228 if (_attributes->is_no_reg_mask() == false &&
11229 _attributes->get_embedded_opmask_register_specifier() != 0) {
11230 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11231 }
11232
11233 emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11234 }
11235
11236 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11237 bool vex_r = (xreg_enc & 8) == 8;
11238 bool vex_b = adr.base_needs_rex();
11239 bool vex_x;
11240 if (adr.isxmmindex()) {
11241 vex_x = adr.xmmindex_needs_rex();
11242 } else {
11243 vex_x = adr.index_needs_rex();
11244 }
11245 set_attributes(attributes);
11246 attributes->set_current_assembler(this);
11247
11248 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11249 // is allowed in legacy mode and has resources which will fit in it.
11250 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11251 if (!attributes->is_legacy_mode()) {
11252 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11253 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11254 attributes->set_is_legacy_mode();
11255 }
11256 }
11257 }
11258
11259 if (UseAVX > 2) {
11260 assert(((!attributes->uses_vl()) ||
11261 (attributes->get_vector_len() == AVX_512bit) ||
11262 (!_legacy_mode_vl) ||
11263 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11264 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11265 }
11266
11267 clear_managed();
11268 if (UseAVX > 2 && !attributes->is_legacy_mode())
11269 {
11270 bool evex_r = (xreg_enc >= 16);
11271 bool evex_v;
11272 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11273 if (adr.isxmmindex()) {
11274 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11275 } else {
11276 evex_v = (nds_enc >= 16);
11277 }
11278 attributes->set_is_evex_instruction();
11279 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11280 } else {
11281 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11282 attributes->set_rex_vex_w(false);
11283 }
11284 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11285 }
11286 }
11287
11288 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11289 bool vex_r = (dst_enc & 8) == 8;
11290 bool vex_b = (src_enc & 8) == 8;
11291 bool vex_x = false;
11292 set_attributes(attributes);
11293 attributes->set_current_assembler(this);
11294
11295 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11296 // is allowed in legacy mode and has resources which will fit in it.
11297 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11298 if (!attributes->is_legacy_mode()) {
11299 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11300 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11301 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11302 attributes->set_is_legacy_mode();
11303 }
11304 }
11305 }
11306
11307 if (UseAVX > 2) {
11308 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11309 // Instruction with uses_vl true are vector instructions
11310 // All the vector instructions with AVX_512bit length can have legacy_mode as false
11311 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11312 // Rest all should have legacy_mode set as true
11313 assert(((!attributes->uses_vl()) ||
11314 (attributes->get_vector_len() == AVX_512bit) ||
11315 (!_legacy_mode_vl) ||
11316 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11317 // Instruction with legacy_mode true should have dst, nds and src < 15
11318 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11319 }
11320
11321 clear_managed();
11322 if (UseAVX > 2 && !attributes->is_legacy_mode())
11323 {
11324 bool evex_r = (dst_enc >= 16);
11325 bool evex_v = (nds_enc >= 16);
11326 // can use vex_x as bank extender on rm encoding
11327 vex_x = (src_enc >= 16);
11328 attributes->set_is_evex_instruction();
11329 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11330 } else {
11331 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11332 attributes->set_rex_vex_w(false);
11333 }
11334 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11335 }
11336
11337 // return modrm byte components for operands
11338 return (((dst_enc & 7) << 3) | (src_enc & 7));
11339 }
11340
11341
11342 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11343 VexOpcode opc, InstructionAttr *attributes) {
11344 if (UseAVX > 0) {
11345 int xreg_enc = xreg->encoding();
11346 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11347 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11348 } else {
11349 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11350 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11351 }
11352 }
11353
11354 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11355 VexOpcode opc, InstructionAttr *attributes) {
11356 int dst_enc = dst->encoding();
11357 int src_enc = src->encoding();
11358 if (UseAVX > 0) {
11359 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11360 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11361 } else {
11362 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11363 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11364 }
11365 }
11366
11367 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11368 assert(VM_Version::supports_avx(), "");
11369 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11370 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11371 emit_int16(0x5F, (0xC0 | encode));
11372 }
11373
11374 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11375 assert(VM_Version::supports_avx(), "");
11376 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11377 attributes.set_rex_vex_w_reverted();
11378 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11379 emit_int16(0x5F, (0xC0 | encode));
11380 }
11381
11382 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11383 assert(VM_Version::supports_avx(), "");
11384 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11385 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11386 emit_int16(0x5D, (0xC0 | encode));
11387 }
11388
11389 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11390 assert(VM_Version::supports_avx(), "");
11391 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11392 attributes.set_rex_vex_w_reverted();
11393 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11394 emit_int16(0x5D, (0xC0 | encode));
11395 }
11396
11397 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11398 assert(VM_Version::supports_avx(), "");
11399 assert(vector_len <= AVX_256bit, "");
11400 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11401 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11402 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11403 }
11404
11405 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11406 assert(VM_Version::supports_avx(), "");
11407 assert(vector_len <= AVX_256bit, "");
11408 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11409 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11410 int src2_enc = src2->encoding();
11411 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11412 }
11413
11414 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11415 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11416 assert(vector_len <= AVX_256bit, "");
11417 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11418 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11419 int src2_enc = src2->encoding();
11420 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11421 }
11422
11423 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11424 assert(VM_Version::supports_avx2(), "");
11425 assert(vector_len <= AVX_256bit, "");
11426 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11427 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11428 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11429 }
11430
11431 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11432 assert(VM_Version::supports_avx(), "");
11433 assert(vector_len <= AVX_256bit, "");
11434 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11435 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11436 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11437 }
11438
11439 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11440 ComparisonPredicateFP comparison, int vector_len) {
11441 assert(VM_Version::supports_evex(), "");
11442 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11443 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11444 attributes.set_is_evex_instruction();
11445 attributes.set_embedded_opmask_register_specifier(mask);
11446 attributes.reset_is_clear_context();
11447 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11448 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11449 }
11450
11451 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11452 ComparisonPredicateFP comparison, int vector_len) {
11453 assert(VM_Version::supports_evex(), "");
11454 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11455 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11456 attributes.set_is_evex_instruction();
11457 attributes.set_embedded_opmask_register_specifier(mask);
11458 attributes.reset_is_clear_context();
11459 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11460 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11461 }
11462
11463 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11464 assert(VM_Version::supports_sse4_1(), "");
11465 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11466 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11467 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11468 emit_int16(0x14, (0xC0 | encode));
11469 }
11470
11471 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11472 assert(VM_Version::supports_sse4_1(), "");
11473 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11474 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11475 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11476 emit_int16(0x15, (0xC0 | encode));
11477 }
11478
11479 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11480 assert(VM_Version::supports_sse4_1(), "");
11481 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11482 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11483 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11484 emit_int16(0x10, (0xC0 | encode));
11485 }
11486
11487 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11488 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11489 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11490 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11491 int src2_enc = src2->encoding();
11492 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11493 }
11494
11495 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11496 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11497 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11498 emit_int24(0x0C, (0xC0 | encode), imm8);
11499 }
11500
11501 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11502 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11503 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11504 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11505 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11506 emit_int16(0x64, (0xC0 | encode));
11507 }
11508
11509 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11510 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11511 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11512 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11513 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11514 emit_int16(0x65, (0xC0 | encode));
11515 }
11516
11517 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11518 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11519 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11520 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11521 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11522 emit_int16(0x66, (0xC0 | encode));
11523 }
11524
11525 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11526 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11527 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11528 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11530 emit_int16(0x37, (0xC0 | encode));
11531 }
11532
11533 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11534 int comparison, bool is_signed, int vector_len) {
11535 assert(VM_Version::supports_evex(), "");
11536 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11537 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11538 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11539 attributes.set_is_evex_instruction();
11540 attributes.set_embedded_opmask_register_specifier(mask);
11541 attributes.reset_is_clear_context();
11542 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11543 int opcode = is_signed ? 0x1F : 0x1E;
11544 emit_int24(opcode, (0xC0 | encode), comparison);
11545 }
11546
11547 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11548 int comparison, bool is_signed, int vector_len) {
11549 assert(VM_Version::supports_evex(), "");
11550 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11551 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11552 InstructionMark im(this);
11553 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11554 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11555 attributes.set_is_evex_instruction();
11556 attributes.set_embedded_opmask_register_specifier(mask);
11557 attributes.reset_is_clear_context();
11558 int dst_enc = kdst->encoding();
11559 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11560 int opcode = is_signed ? 0x1F : 0x1E;
11561 emit_int8((unsigned char)opcode);
11562 emit_operand(as_Register(dst_enc), src, 1);
11563 emit_int8((unsigned char)comparison);
11564 }
11565
11566 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11567 int comparison, bool is_signed, int vector_len) {
11568 assert(VM_Version::supports_evex(), "");
11569 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11570 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11571 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11572 attributes.set_is_evex_instruction();
11573 attributes.set_embedded_opmask_register_specifier(mask);
11574 attributes.reset_is_clear_context();
11575 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11576 int opcode = is_signed ? 0x1F : 0x1E;
11577 emit_int24(opcode, (0xC0 | encode), comparison);
11578 }
11579
11580 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11581 int comparison, bool is_signed, int vector_len) {
11582 assert(VM_Version::supports_evex(), "");
11583 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11584 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11585 InstructionMark im(this);
11586 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11587 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11588 attributes.set_is_evex_instruction();
11589 attributes.set_embedded_opmask_register_specifier(mask);
11590 attributes.reset_is_clear_context();
11591 int dst_enc = kdst->encoding();
11592 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11593 int opcode = is_signed ? 0x1F : 0x1E;
11594 emit_int8((unsigned char)opcode);
11595 emit_operand(as_Register(dst_enc), src, 1);
11596 emit_int8((unsigned char)comparison);
11597 }
11598
11599 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11600 int comparison, bool is_signed, int vector_len) {
11601 assert(VM_Version::supports_evex(), "");
11602 assert(VM_Version::supports_avx512bw(), "");
11603 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11604 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11605 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11606 attributes.set_is_evex_instruction();
11607 attributes.set_embedded_opmask_register_specifier(mask);
11608 attributes.reset_is_clear_context();
11609 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11610 int opcode = is_signed ? 0x3F : 0x3E;
11611 emit_int24(opcode, (0xC0 | encode), comparison);
11612 }
11613
11614 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11615 int comparison, bool is_signed, int vector_len) {
11616 assert(VM_Version::supports_evex(), "");
11617 assert(VM_Version::supports_avx512bw(), "");
11618 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11619 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11620 InstructionMark im(this);
11621 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11622 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11623 attributes.set_is_evex_instruction();
11624 attributes.set_embedded_opmask_register_specifier(mask);
11625 attributes.reset_is_clear_context();
11626 int dst_enc = kdst->encoding();
11627 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11628 int opcode = is_signed ? 0x3F : 0x3E;
11629 emit_int8((unsigned char)opcode);
11630 emit_operand(as_Register(dst_enc), src, 1);
11631 emit_int8((unsigned char)comparison);
11632 }
11633
11634 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11635 int comparison, bool is_signed, int vector_len) {
11636 assert(VM_Version::supports_evex(), "");
11637 assert(VM_Version::supports_avx512bw(), "");
11638 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11639 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11640 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11641 attributes.set_is_evex_instruction();
11642 attributes.set_embedded_opmask_register_specifier(mask);
11643 attributes.reset_is_clear_context();
11644 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11645 int opcode = is_signed ? 0x3F : 0x3E;
11646 emit_int24(opcode, (0xC0 | encode), comparison);
11647 }
11648
11649 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11650 int comparison, bool is_signed, int vector_len) {
11651 assert(VM_Version::supports_evex(), "");
11652 assert(VM_Version::supports_avx512bw(), "");
11653 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11654 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11655 InstructionMark im(this);
11656 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11657 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11658 attributes.set_is_evex_instruction();
11659 attributes.set_embedded_opmask_register_specifier(mask);
11660 attributes.reset_is_clear_context();
11661 int dst_enc = kdst->encoding();
11662 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11663 int opcode = is_signed ? 0x3F : 0x3E;
11664 emit_int8((unsigned char)opcode);
11665 emit_operand(as_Register(dst_enc), src, 1);
11666 emit_int8((unsigned char)comparison);
11667 }
11668
11669 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11670 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11671 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11672 attributes.set_is_evex_instruction();
11673 attributes.set_embedded_opmask_register_specifier(mask);
11674 if (merge) {
11675 attributes.reset_is_clear_context();
11676 }
11677 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11678 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11679 }
11680
11681 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11682 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11683 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11684 attributes.set_is_evex_instruction();
11685 attributes.set_embedded_opmask_register_specifier(mask);
11686 if (merge) {
11687 attributes.reset_is_clear_context();
11688 }
11689 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11690 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11691 }
11692
11693 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11694 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11695 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11696 attributes.set_is_evex_instruction();
11697 attributes.set_embedded_opmask_register_specifier(mask);
11698 if (merge) {
11699 attributes.reset_is_clear_context();
11700 }
11701 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11702 emit_int16(0x14, (0xC0 | encode));
11703 }
11704
11705 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11706 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11707 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11708 attributes.set_is_evex_instruction();
11709 attributes.set_embedded_opmask_register_specifier(mask);
11710 if (merge) {
11711 attributes.reset_is_clear_context();
11712 }
11713 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11714 emit_int16(0x14, (0xC0 | encode));
11715 }
11716
11717 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11718 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11719 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11720 attributes.set_is_evex_instruction();
11721 attributes.set_embedded_opmask_register_specifier(mask);
11722 if (merge) {
11723 attributes.reset_is_clear_context();
11724 }
11725 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11726 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11727 }
11728
11729 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11730 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11731 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11732 attributes.set_is_evex_instruction();
11733 attributes.set_embedded_opmask_register_specifier(mask);
11734 if (merge) {
11735 attributes.reset_is_clear_context();
11736 }
11737 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11738 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11739 }
11740
11741 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11742 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11743 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11744 attributes.set_is_evex_instruction();
11745 attributes.set_embedded_opmask_register_specifier(mask);
11746 if (merge) {
11747 attributes.reset_is_clear_context();
11748 }
11749 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11750 emit_int16(0x15, (0xC0 | encode));
11751 }
11752
11753 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11754 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11755 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11756 attributes.set_is_evex_instruction();
11757 attributes.set_embedded_opmask_register_specifier(mask);
11758 if (merge) {
11759 attributes.reset_is_clear_context();
11760 }
11761 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11762 emit_int16(0x15, (0xC0 | encode));
11763 }
11764
11765 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11766 assert(VM_Version::supports_avx(), "");
11767 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11769 int mask_enc = mask->encoding();
11770 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11771 }
11772
11773 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11774 assert(VM_Version::supports_evex(), "");
11775 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11776 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11777 attributes.set_is_evex_instruction();
11778 attributes.set_embedded_opmask_register_specifier(mask);
11779 if (merge) {
11780 attributes.reset_is_clear_context();
11781 }
11782 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11783 emit_int16(0x65, (0xC0 | encode));
11784 }
11785
11786 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11787 assert(VM_Version::supports_evex(), "");
11788 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11789 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11790 attributes.set_is_evex_instruction();
11791 attributes.set_embedded_opmask_register_specifier(mask);
11792 if (merge) {
11793 attributes.reset_is_clear_context();
11794 }
11795 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11796 emit_int16(0x65, (0xC0 | encode));
11797 }
11798
11799 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11800 assert(VM_Version::supports_evex(), "");
11801 assert(VM_Version::supports_avx512bw(), "");
11802 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11803 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11804 attributes.set_is_evex_instruction();
11805 attributes.set_embedded_opmask_register_specifier(mask);
11806 if (merge) {
11807 attributes.reset_is_clear_context();
11808 }
11809 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11810 emit_int16(0x66, (0xC0 | encode));
11811 }
11812
11813 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11814 assert(VM_Version::supports_evex(), "");
11815 assert(VM_Version::supports_avx512bw(), "");
11816 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11817 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11818 attributes.set_is_evex_instruction();
11819 attributes.set_embedded_opmask_register_specifier(mask);
11820 if (merge) {
11821 attributes.reset_is_clear_context();
11822 }
11823 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11824 emit_int16(0x66, (0xC0 | encode));
11825 }
11826
11827 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11828 assert(VM_Version::supports_evex(), "");
11829 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11830 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11831 attributes.set_is_evex_instruction();
11832 attributes.set_embedded_opmask_register_specifier(mask);
11833 if (merge) {
11834 attributes.reset_is_clear_context();
11835 }
11836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11837 emit_int16(0x64, (0xC0 | encode));
11838 }
11839
11840 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11841 assert(VM_Version::supports_evex(), "");
11842 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11843 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11844 attributes.set_is_evex_instruction();
11845 attributes.set_embedded_opmask_register_specifier(mask);
11846 if (merge) {
11847 attributes.reset_is_clear_context();
11848 }
11849 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11850 emit_int16(0x64, (0xC0 | encode));
11851 }
11852
11853 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11854 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11855 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11856 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11857 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11858 }
11859
11860 void Assembler::pextl(Register dst, Register src1, Register src2) {
11861 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11862 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11863 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11864 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11865 }
11866
11867 void Assembler::pdepl(Register dst, Register src1, Register src2) {
11868 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11869 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11870 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11871 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11872 }
11873
11874 void Assembler::pextq(Register dst, Register src1, Register src2) {
11875 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11876 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11877 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11878 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11879 }
11880
11881 void Assembler::pdepq(Register dst, Register src1, Register src2) {
11882 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11883 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11884 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11885 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11886 }
11887
11888 void Assembler::pextl(Register dst, Register src1, Address src2) {
11889 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11890 InstructionMark im(this);
11891 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11892 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11893 emit_int8((unsigned char)0xF5);
11894 emit_operand(dst, src2, 0);
11895 }
11896
11897 void Assembler::pdepl(Register dst, Register src1, Address src2) {
11898 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11899 InstructionMark im(this);
11900 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11901 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11902 emit_int8((unsigned char)0xF5);
11903 emit_operand(dst, src2, 0);
11904 }
11905
11906 void Assembler::pextq(Register dst, Register src1, Address src2) {
11907 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11908 InstructionMark im(this);
11909 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11910 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11911 emit_int8((unsigned char)0xF5);
11912 emit_operand(dst, src2, 0);
11913 }
11914
11915 void Assembler::pdepq(Register dst, Register src1, Address src2) {
11916 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11917 InstructionMark im(this);
11918 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11919 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11920 emit_int8((unsigned char)0xF5);
11921 emit_operand(dst, src2, 0);
11922 }
11923
11924 void Assembler::sarxl(Register dst, Register src1, Register src2) {
11925 assert(VM_Version::supports_bmi2(), "");
11926 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11927 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11928 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11929 }
11930
11931 void Assembler::sarxl(Register dst, Address src1, Register src2) {
11932 assert(VM_Version::supports_bmi2(), "");
11933 InstructionMark im(this);
11934 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11935 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11936 emit_int8((unsigned char)0xF7);
11937 emit_operand(dst, src1, 0);
11938 }
11939
11940 void Assembler::sarxq(Register dst, Register src1, Register src2) {
11941 assert(VM_Version::supports_bmi2(), "");
11942 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11943 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11944 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11945 }
11946
11947 void Assembler::sarxq(Register dst, Address src1, Register src2) {
11948 assert(VM_Version::supports_bmi2(), "");
11949 InstructionMark im(this);
11950 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11951 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11952 emit_int8((unsigned char)0xF7);
11953 emit_operand(dst, src1, 0);
11954 }
11955
11956 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11957 assert(VM_Version::supports_bmi2(), "");
11958 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11959 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11960 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11961 }
11962
11963 void Assembler::shlxl(Register dst, Address src1, Register src2) {
11964 assert(VM_Version::supports_bmi2(), "");
11965 InstructionMark im(this);
11966 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11967 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11968 emit_int8((unsigned char)0xF7);
11969 emit_operand(dst, src1, 0);
11970 }
11971
11972 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11973 assert(VM_Version::supports_bmi2(), "");
11974 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11975 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11976 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11977 }
11978
11979 void Assembler::shlxq(Register dst, Address src1, Register src2) {
11980 assert(VM_Version::supports_bmi2(), "");
11981 InstructionMark im(this);
11982 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11983 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11984 emit_int8((unsigned char)0xF7);
11985 emit_operand(dst, src1, 0);
11986 }
11987
11988 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11989 assert(VM_Version::supports_bmi2(), "");
11990 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11991 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11992 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11993 }
11994
11995 void Assembler::shrxl(Register dst, Address src1, Register src2) {
11996 assert(VM_Version::supports_bmi2(), "");
11997 InstructionMark im(this);
11998 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11999 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12000 emit_int8((unsigned char)0xF7);
12001 emit_operand(dst, src1, 0);
12002 }
12003
12004 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12005 assert(VM_Version::supports_bmi2(), "");
12006 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12007 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12008 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12009 }
12010
12011 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12012 assert(VM_Version::supports_bmi2(), "");
12013 InstructionMark im(this);
12014 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12015 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12016 emit_int8((unsigned char)0xF7);
12017 emit_operand(dst, src1, 0);
12018 }
12019
12020 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12021 assert(VM_Version::supports_avx512vldq(), "");
12022 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12023 attributes.set_is_evex_instruction();
12024 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12025 emit_int16(0x39, (0xC0 | encode));
12026 }
12027
12028 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12029 assert(VM_Version::supports_avx512vldq(), "");
12030 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12031 attributes.set_is_evex_instruction();
12032 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12033 emit_int16(0x39, (0xC0 | encode));
12034 }
12035
12036 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12037 assert(VM_Version::supports_avx512vlbw(), "");
12038 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12039 attributes.set_is_evex_instruction();
12040 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12041 emit_int16(0x29, (0xC0 | encode));
12042 }
12043
12044 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12045 assert(VM_Version::supports_avx512vlbw(), "");
12046 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12047 attributes.set_is_evex_instruction();
12048 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12049 emit_int16(0x29, (0xC0 | encode));
12050 }
12051
12052 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12053 assert(VM_Version::supports_avx512vldq(), "");
12054 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12055 attributes.set_is_evex_instruction();
12056 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12057 emit_int16(0x38, (0xC0 | encode));
12058 }
12059
12060 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12061 assert(VM_Version::supports_avx512vldq(), "");
12062 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12063 attributes.set_is_evex_instruction();
12064 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12065 emit_int16(0x38, (0xC0 | encode));
12066 }
12067
12068 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12069 assert(VM_Version::supports_avx512vlbw(), "");
12070 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12071 attributes.set_is_evex_instruction();
12072 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12073 emit_int16(0x28, (0xC0 | encode));
12074 }
12075
12076 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12077 assert(VM_Version::supports_avx512vlbw(), "");
12078 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12079 attributes.set_is_evex_instruction();
12080 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12081 emit_int16(0x28, (0xC0 | encode));
12082 }
12083
12084 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12085 assert(VM_Version::supports_avx512_vbmi2(), "");
12086 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12088 attributes.set_embedded_opmask_register_specifier(mask);
12089 attributes.set_is_evex_instruction();
12090 if (merge) {
12091 attributes.reset_is_clear_context();
12092 }
12093 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12094 emit_int16((unsigned char)0x63, (0xC0 | encode));
12095 }
12096
12097 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12098 assert(VM_Version::supports_avx512_vbmi2(), "");
12099 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12100 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12101 attributes.set_embedded_opmask_register_specifier(mask);
12102 attributes.set_is_evex_instruction();
12103 if (merge) {
12104 attributes.reset_is_clear_context();
12105 }
12106 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12107 emit_int16((unsigned char)0x63, (0xC0 | encode));
12108 }
12109
12110 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12111 assert(VM_Version::supports_evex(), "");
12112 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12113 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12114 attributes.set_embedded_opmask_register_specifier(mask);
12115 attributes.set_is_evex_instruction();
12116 if (merge) {
12117 attributes.reset_is_clear_context();
12118 }
12119 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12120 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12121 }
12122
12123 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12124 assert(VM_Version::supports_evex(), "");
12125 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12126 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12127 attributes.set_embedded_opmask_register_specifier(mask);
12128 attributes.set_is_evex_instruction();
12129 if (merge) {
12130 attributes.reset_is_clear_context();
12131 }
12132 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12133 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12134 }
12135
12136 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12137 assert(VM_Version::supports_evex(), "");
12138 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12139 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12140 attributes.set_embedded_opmask_register_specifier(mask);
12141 attributes.set_is_evex_instruction();
12142 if (merge) {
12143 attributes.reset_is_clear_context();
12144 }
12145 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12146 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12147 }
12148
12149 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12150 assert(VM_Version::supports_evex(), "");
12151 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12152 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12153 attributes.set_embedded_opmask_register_specifier(mask);
12154 attributes.set_is_evex_instruction();
12155 if (merge) {
12156 attributes.reset_is_clear_context();
12157 }
12158 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12159 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12160 }
12161
12162 #ifndef _LP64
12163
12164 void Assembler::incl(Register dst) {
12165 // Don't use it directly. Use MacroAssembler::incrementl() instead.
12166 emit_int8(0x40 | dst->encoding());
12167 }
12168
12169 void Assembler::lea(Register dst, Address src) {
12170 leal(dst, src);
12171 }
12172
12173 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12174 InstructionMark im(this);
12175 emit_int8((unsigned char)0xC7);
12176 emit_operand(rax, dst, 4);
12177 emit_data((int)imm32, rspec, 0);
12178 }
12179
12180 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12181 InstructionMark im(this);
12182 int encode = prefix_and_encode(dst->encoding());
12183 emit_int8((0xB8 | encode));
12184 emit_data((int)imm32, rspec, 0);
12185 }
12186
12187 void Assembler::popa() { // 32bit
12188 emit_int8(0x61);
12189 }
12190
12191 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12192 InstructionMark im(this);
12193 emit_int8(0x68);
12194 emit_data(imm32, rspec, 0);
12195 }
12196
12197 void Assembler::pusha() { // 32bit
12198 emit_int8(0x60);
12199 }
12200
12201 void Assembler::set_byte_if_not_zero(Register dst) {
12202 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12203 }
12204
12205 #else // LP64
12206
12207 // 64bit only pieces of the assembler
12208
12209 void Assembler::set_byte_if_not_zero(Register dst) {
12210 int enc = prefix_and_encode(dst->encoding(), true);
12211 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12212 }
12213
12214 // This should only be used by 64bit instructions that can use rip-relative
12215 // it cannot be used by instructions that want an immediate value.
12216
12217 // Determine whether an address is always reachable in rip-relative addressing mode
12218 // when accessed from the code cache.
12219 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12220 switch (reloc_type) {
12221 // This should be rip-relative and easily reachable.
12222 case relocInfo::internal_word_type: {
12223 return true;
12224 }
12225 // This should be rip-relative within the code cache and easily
12226 // reachable until we get huge code caches. (At which point
12227 // IC code is going to have issues).
12228 case relocInfo::virtual_call_type:
12229 case relocInfo::opt_virtual_call_type:
12230 case relocInfo::static_call_type:
12231 case relocInfo::static_stub_type: {
12232 return true;
12233 }
12234 case relocInfo::runtime_call_type:
12235 case relocInfo::external_word_type:
12236 case relocInfo::poll_return_type: // these are really external_word but need special
12237 case relocInfo::poll_type: { // relocs to identify them
12238 return CodeCache::contains(target);
12239 }
12240 default: {
12241 return false;
12242 }
12243 }
12244 }
12245
12246 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12247 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12248 if (is_always_reachable(target, reloc_type)) {
12249 return true;
12250 }
12251 switch (reloc_type) {
12252 // None will force a 64bit literal to the code stream. Likely a placeholder
12253 // for something that will be patched later and we need to certain it will
12254 // always be reachable.
12255 case relocInfo::none: {
12256 return false;
12257 }
12258 case relocInfo::runtime_call_type:
12259 case relocInfo::external_word_type:
12260 case relocInfo::poll_return_type: // these are really external_word but need special
12261 case relocInfo::poll_type: { // relocs to identify them
12262 assert(!CodeCache::contains(target), "always reachable");
12263 if (ForceUnreachable) {
12264 return false; // stress the correction code
12265 }
12266 // For external_word_type/runtime_call_type if it is reachable from where we
12267 // are now (possibly a temp buffer) and where we might end up
12268 // anywhere in the code cache then we are always reachable.
12269 // This would have to change if we ever save/restore shared code to be more pessimistic.
12270 // Code buffer has to be allocated in the code cache, so check against
12271 // code cache boundaries cover that case.
12272 //
12273 // In rip-relative addressing mode, an effective address is formed by adding displacement
12274 // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12275 // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12276 // to account for that.
12277 return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12278 Assembler::is_simm32(target - CodeCache::high_bound());
12279 }
12280 default: {
12281 return false;
12282 }
12283 }
12284 }
12285
12286 bool Assembler::reachable(AddressLiteral adr) {
12287 assert(CodeCache::contains(pc()), "required");
12288 if (adr.is_lval()) {
12289 return false;
12290 }
12291 return is_reachable(adr.target(), adr.reloc());
12292 }
12293
12294 bool Assembler::always_reachable(AddressLiteral adr) {
12295 assert(CodeCache::contains(pc()), "required");
12296 if (adr.is_lval()) {
12297 return false;
12298 }
12299 return is_always_reachable(adr.target(), adr.reloc());
12300 }
12301
12302 void Assembler::emit_data64(jlong data,
12303 relocInfo::relocType rtype,
12304 int format) {
12305 if (rtype == relocInfo::none) {
12306 emit_int64(data);
12307 } else {
12308 emit_data64(data, Relocation::spec_simple(rtype), format);
12309 }
12310 }
12311
12312 void Assembler::emit_data64(jlong data,
12313 RelocationHolder const& rspec,
12314 int format) {
12315 assert(imm_operand == 0, "default format must be immediate in this file");
12316 assert(imm_operand == format, "must be immediate");
12317 assert(inst_mark() != NULL, "must be inside InstructionMark");
12318 // Do not use AbstractAssembler::relocate, which is not intended for
12319 // embedded words. Instead, relocate to the enclosing instruction.
12320 code_section()->relocate(inst_mark(), rspec, format);
12321 #ifdef ASSERT
12322 check_relocation(rspec, format);
12323 #endif
12324 emit_int64(data);
12325 }
12326
12327 void Assembler::prefix(Register reg) {
12328 if (reg->encoding() >= 8) {
12329 prefix(REX_B);
12330 }
12331 }
12332
12333 void Assembler::prefix(Register dst, Register src, Prefix p) {
12334 if (src->encoding() >= 8) {
12335 p = (Prefix)(p | REX_B);
12336 }
12337 if (dst->encoding() >= 8) {
12338 p = (Prefix)(p | REX_R);
12339 }
12340 if (p != Prefix_EMPTY) {
12341 // do not generate an empty prefix
12342 prefix(p);
12343 }
12344 }
12345
12346 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12347 if (adr.base_needs_rex()) {
12348 if (adr.index_needs_rex()) {
12349 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12350 } else {
12351 prefix(REX_B);
12352 }
12353 } else {
12354 if (adr.index_needs_rex()) {
12355 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12356 }
12357 }
12358 if (dst->encoding() >= 8) {
12359 p = (Prefix)(p | REX_R);
12360 }
12361 if (p != Prefix_EMPTY) {
12362 // do not generate an empty prefix
12363 prefix(p);
12364 }
12365 }
12366
12367 void Assembler::prefix(Address adr) {
12368 if (adr.base_needs_rex()) {
12369 if (adr.index_needs_rex()) {
12370 prefix(REX_XB);
12371 } else {
12372 prefix(REX_B);
12373 }
12374 } else {
12375 if (adr.index_needs_rex()) {
12376 prefix(REX_X);
12377 }
12378 }
12379 }
12380
12381 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12382 if (reg->encoding() < 8) {
12383 if (adr.base_needs_rex()) {
12384 if (adr.index_needs_rex()) {
12385 prefix(REX_XB);
12386 } else {
12387 prefix(REX_B);
12388 }
12389 } else {
12390 if (adr.index_needs_rex()) {
12391 prefix(REX_X);
12392 } else if (byteinst && reg->encoding() >= 4) {
12393 prefix(REX);
12394 }
12395 }
12396 } else {
12397 if (adr.base_needs_rex()) {
12398 if (adr.index_needs_rex()) {
12399 prefix(REX_RXB);
12400 } else {
12401 prefix(REX_RB);
12402 }
12403 } else {
12404 if (adr.index_needs_rex()) {
12405 prefix(REX_RX);
12406 } else {
12407 prefix(REX_R);
12408 }
12409 }
12410 }
12411 }
12412
12413 void Assembler::prefix(Address adr, XMMRegister reg) {
12414 if (reg->encoding() < 8) {
12415 if (adr.base_needs_rex()) {
12416 if (adr.index_needs_rex()) {
12417 prefix(REX_XB);
12418 } else {
12419 prefix(REX_B);
12420 }
12421 } else {
12422 if (adr.index_needs_rex()) {
12423 prefix(REX_X);
12424 }
12425 }
12426 } else {
12427 if (adr.base_needs_rex()) {
12428 if (adr.index_needs_rex()) {
12429 prefix(REX_RXB);
12430 } else {
12431 prefix(REX_RB);
12432 }
12433 } else {
12434 if (adr.index_needs_rex()) {
12435 prefix(REX_RX);
12436 } else {
12437 prefix(REX_R);
12438 }
12439 }
12440 }
12441 }
12442
12443 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12444 if (reg_enc >= 8) {
12445 prefix(REX_B);
12446 reg_enc -= 8;
12447 } else if (byteinst && reg_enc >= 4) {
12448 prefix(REX);
12449 }
12450 return reg_enc;
12451 }
12452
12453 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12454 if (dst_enc < 8) {
12455 if (src_enc >= 8) {
12456 prefix(REX_B);
12457 src_enc -= 8;
12458 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12459 prefix(REX);
12460 }
12461 } else {
12462 if (src_enc < 8) {
12463 prefix(REX_R);
12464 } else {
12465 prefix(REX_RB);
12466 src_enc -= 8;
12467 }
12468 dst_enc -= 8;
12469 }
12470 return dst_enc << 3 | src_enc;
12471 }
12472
12473 int8_t Assembler::get_prefixq(Address adr) {
12474 int8_t prfx = get_prefixq(adr, rax);
12475 assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12476 return prfx;
12477 }
12478
12479 int8_t Assembler::get_prefixq(Address adr, Register src) {
12480 int8_t prfx = (int8_t)(REX_W +
12481 ((int)adr.base_needs_rex()) +
12482 ((int)adr.index_needs_rex() << 1) +
12483 ((int)(src->encoding() >= 8) << 2));
12484 #ifdef ASSERT
12485 if (src->encoding() < 8) {
12486 if (adr.base_needs_rex()) {
12487 if (adr.index_needs_rex()) {
12488 assert(prfx == REX_WXB, "must be");
12489 } else {
12490 assert(prfx == REX_WB, "must be");
12491 }
12492 } else {
12493 if (adr.index_needs_rex()) {
12494 assert(prfx == REX_WX, "must be");
12495 } else {
12496 assert(prfx == REX_W, "must be");
12497 }
12498 }
12499 } else {
12500 if (adr.base_needs_rex()) {
12501 if (adr.index_needs_rex()) {
12502 assert(prfx == REX_WRXB, "must be");
12503 } else {
12504 assert(prfx == REX_WRB, "must be");
12505 }
12506 } else {
12507 if (adr.index_needs_rex()) {
12508 assert(prfx == REX_WRX, "must be");
12509 } else {
12510 assert(prfx == REX_WR, "must be");
12511 }
12512 }
12513 }
12514 #endif
12515 return prfx;
12516 }
12517
12518 void Assembler::prefixq(Address adr) {
12519 emit_int8(get_prefixq(adr));
12520 }
12521
12522 void Assembler::prefixq(Address adr, Register src) {
12523 emit_int8(get_prefixq(adr, src));
12524 }
12525
12526 void Assembler::prefixq(Address adr, XMMRegister src) {
12527 if (src->encoding() < 8) {
12528 if (adr.base_needs_rex()) {
12529 if (adr.index_needs_rex()) {
12530 prefix(REX_WXB);
12531 } else {
12532 prefix(REX_WB);
12533 }
12534 } else {
12535 if (adr.index_needs_rex()) {
12536 prefix(REX_WX);
12537 } else {
12538 prefix(REX_W);
12539 }
12540 }
12541 } else {
12542 if (adr.base_needs_rex()) {
12543 if (adr.index_needs_rex()) {
12544 prefix(REX_WRXB);
12545 } else {
12546 prefix(REX_WRB);
12547 }
12548 } else {
12549 if (adr.index_needs_rex()) {
12550 prefix(REX_WRX);
12551 } else {
12552 prefix(REX_WR);
12553 }
12554 }
12555 }
12556 }
12557
12558 int Assembler::prefixq_and_encode(int reg_enc) {
12559 if (reg_enc < 8) {
12560 prefix(REX_W);
12561 } else {
12562 prefix(REX_WB);
12563 reg_enc -= 8;
12564 }
12565 return reg_enc;
12566 }
12567
12568 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12569 if (dst_enc < 8) {
12570 if (src_enc < 8) {
12571 prefix(REX_W);
12572 } else {
12573 prefix(REX_WB);
12574 src_enc -= 8;
12575 }
12576 } else {
12577 if (src_enc < 8) {
12578 prefix(REX_WR);
12579 } else {
12580 prefix(REX_WRB);
12581 src_enc -= 8;
12582 }
12583 dst_enc -= 8;
12584 }
12585 return dst_enc << 3 | src_enc;
12586 }
12587
12588 void Assembler::adcq(Register dst, int32_t imm32) {
12589 (void) prefixq_and_encode(dst->encoding());
12590 emit_arith(0x81, 0xD0, dst, imm32);
12591 }
12592
12593 void Assembler::adcq(Register dst, Address src) {
12594 InstructionMark im(this);
12595 emit_int16(get_prefixq(src, dst), 0x13);
12596 emit_operand(dst, src, 0);
12597 }
12598
12599 void Assembler::adcq(Register dst, Register src) {
12600 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12601 emit_arith(0x13, 0xC0, dst, src);
12602 }
12603
12604 void Assembler::addq(Address dst, int32_t imm32) {
12605 InstructionMark im(this);
12606 prefixq(dst);
12607 emit_arith_operand(0x81, rax, dst, imm32);
12608 }
12609
12610 void Assembler::addq(Address dst, Register src) {
12611 InstructionMark im(this);
12612 emit_int16(get_prefixq(dst, src), 0x01);
12613 emit_operand(src, dst, 0);
12614 }
12615
12616 void Assembler::addq(Register dst, int32_t imm32) {
12617 (void) prefixq_and_encode(dst->encoding());
12618 emit_arith(0x81, 0xC0, dst, imm32);
12619 }
12620
12621 void Assembler::addq(Register dst, Address src) {
12622 InstructionMark im(this);
12623 emit_int16(get_prefixq(src, dst), 0x03);
12624 emit_operand(dst, src, 0);
12625 }
12626
12627 void Assembler::addq(Register dst, Register src) {
12628 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12629 emit_arith(0x03, 0xC0, dst, src);
12630 }
12631
12632 void Assembler::adcxq(Register dst, Register src) {
12633 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12634 emit_int8(0x66);
12635 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12636 emit_int32(0x0F,
12637 0x38,
12638 (unsigned char)0xF6,
12639 (0xC0 | encode));
12640 }
12641
12642 void Assembler::adoxq(Register dst, Register src) {
12643 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12644 emit_int8((unsigned char)0xF3);
12645 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12646 emit_int32(0x0F,
12647 0x38,
12648 (unsigned char)0xF6,
12649 (0xC0 | encode));
12650 }
12651
12652 void Assembler::andq(Address dst, int32_t imm32) {
12653 InstructionMark im(this);
12654 prefixq(dst);
12655 emit_arith_operand(0x81, as_Register(4), dst, imm32);
12656 }
12657
12658 void Assembler::andq(Register dst, int32_t imm32) {
12659 (void) prefixq_and_encode(dst->encoding());
12660 emit_arith(0x81, 0xE0, dst, imm32);
12661 }
12662
12663 void Assembler::andq(Register dst, Address src) {
12664 InstructionMark im(this);
12665 emit_int16(get_prefixq(src, dst), 0x23);
12666 emit_operand(dst, src, 0);
12667 }
12668
12669 void Assembler::andq(Register dst, Register src) {
12670 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12671 emit_arith(0x23, 0xC0, dst, src);
12672 }
12673
12674 void Assembler::andq(Address dst, Register src) {
12675 InstructionMark im(this);
12676 emit_int16(get_prefixq(dst, src), 0x21);
12677 emit_operand(src, dst, 0);
12678 }
12679
12680 void Assembler::andnq(Register dst, Register src1, Register src2) {
12681 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12682 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12683 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12684 emit_int16((unsigned char)0xF2, (0xC0 | encode));
12685 }
12686
12687 void Assembler::andnq(Register dst, Register src1, Address src2) {
12688 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12689 InstructionMark im(this);
12690 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12691 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12692 emit_int8((unsigned char)0xF2);
12693 emit_operand(dst, src2, 0);
12694 }
12695
12696 void Assembler::bsfq(Register dst, Register src) {
12697 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12698 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12699 }
12700
12701 void Assembler::bsrq(Register dst, Register src) {
12702 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12703 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12704 }
12705
12706 void Assembler::bswapq(Register reg) {
12707 int encode = prefixq_and_encode(reg->encoding());
12708 emit_int16(0x0F, (0xC8 | encode));
12709 }
12710
12711 void Assembler::blsiq(Register dst, Register src) {
12712 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12713 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12714 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12715 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12716 }
12717
12718 void Assembler::blsiq(Register dst, Address src) {
12719 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12720 InstructionMark im(this);
12721 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12722 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12723 emit_int8((unsigned char)0xF3);
12724 emit_operand(rbx, src, 0);
12725 }
12726
12727 void Assembler::blsmskq(Register dst, Register src) {
12728 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12729 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12730 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12731 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12732 }
12733
12734 void Assembler::blsmskq(Register dst, Address src) {
12735 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12736 InstructionMark im(this);
12737 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12738 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12739 emit_int8((unsigned char)0xF3);
12740 emit_operand(rdx, src, 0);
12741 }
12742
12743 void Assembler::blsrq(Register dst, Register src) {
12744 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12745 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12746 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12747 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12748 }
12749
12750 void Assembler::blsrq(Register dst, Address src) {
12751 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12752 InstructionMark im(this);
12753 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12754 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12755 emit_int8((unsigned char)0xF3);
12756 emit_operand(rcx, src, 0);
12757 }
12758
12759 void Assembler::cdqq() {
12760 emit_int16(REX_W, (unsigned char)0x99);
12761 }
12762
12763 void Assembler::clflush(Address adr) {
12764 assert(VM_Version::supports_clflush(), "should do");
12765 prefix(adr);
12766 emit_int16(0x0F, (unsigned char)0xAE);
12767 emit_operand(rdi, adr, 0);
12768 }
12769
12770 void Assembler::clflushopt(Address adr) {
12771 assert(VM_Version::supports_clflushopt(), "should do!");
12772 // adr should be base reg only with no index or offset
12773 assert(adr.index() == noreg, "index should be noreg");
12774 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12775 assert(adr.disp() == 0, "displacement should be 0");
12776 // instruction prefix is 0x66
12777 emit_int8(0x66);
12778 prefix(adr);
12779 // opcode family is 0x0F 0xAE
12780 emit_int16(0x0F, (unsigned char)0xAE);
12781 // extended opcode byte is 7 == rdi
12782 emit_operand(rdi, adr, 0);
12783 }
12784
12785 void Assembler::clwb(Address adr) {
12786 assert(VM_Version::supports_clwb(), "should do!");
12787 // adr should be base reg only with no index or offset
12788 assert(adr.index() == noreg, "index should be noreg");
12789 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12790 assert(adr.disp() == 0, "displacement should be 0");
12791 // instruction prefix is 0x66
12792 emit_int8(0x66);
12793 prefix(adr);
12794 // opcode family is 0x0f 0xAE
12795 emit_int16(0x0F, (unsigned char)0xAE);
12796 // extended opcode byte is 6 == rsi
12797 emit_operand(rsi, adr, 0);
12798 }
12799
12800 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12801 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12802 emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12803 }
12804
12805 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12806 InstructionMark im(this);
12807 emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12808 emit_operand(dst, src, 0);
12809 }
12810
12811 void Assembler::cmpq(Address dst, int32_t imm32) {
12812 InstructionMark im(this);
12813 prefixq(dst);
12814 emit_arith_operand(0x81, as_Register(7), dst, imm32);
12815 }
12816
12817 void Assembler::cmpq(Register dst, int32_t imm32) {
12818 (void) prefixq_and_encode(dst->encoding());
12819 emit_arith(0x81, 0xF8, dst, imm32);
12820 }
12821
12822 void Assembler::cmpq(Address dst, Register src) {
12823 InstructionMark im(this);
12824 emit_int16(get_prefixq(dst, src), 0x39);
12825 emit_operand(src, dst, 0);
12826 }
12827
12828 void Assembler::cmpq(Register dst, Register src) {
12829 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12830 emit_arith(0x3B, 0xC0, dst, src);
12831 }
12832
12833 void Assembler::cmpq(Register dst, Address src) {
12834 InstructionMark im(this);
12835 emit_int16(get_prefixq(src, dst), 0x3B);
12836 emit_operand(dst, src, 0);
12837 }
12838
12839 void Assembler::cmpxchgq(Register reg, Address adr) {
12840 InstructionMark im(this);
12841 emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12842 emit_operand(reg, adr, 0);
12843 }
12844
12845 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12846 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12847 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12848 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12849 emit_int16(0x2A, (0xC0 | encode));
12850 }
12851
12852 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12853 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12854 InstructionMark im(this);
12855 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12856 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12857 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12858 emit_int8(0x2A);
12859 emit_operand(dst, src, 0);
12860 }
12861
12862 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12863 NOT_LP64(assert(VM_Version::supports_sse(), ""));
12864 InstructionMark im(this);
12865 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12866 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12867 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12868 emit_int8(0x2A);
12869 emit_operand(dst, src, 0);
12870 }
12871
12872 void Assembler::cvttsd2siq(Register dst, Address src) {
12873 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12874 // F2 REX.W 0F 2C /r
12875 // CVTTSD2SI r64, xmm1/m64
12876 InstructionMark im(this);
12877 emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12878 emit_operand(dst, src, 0);
12879 }
12880
12881 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12882 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12883 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12884 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12885 emit_int16(0x2C, (0xC0 | encode));
12886 }
12887
12888 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12889 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12890 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12891 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12892 emit_int16(0x2D, (0xC0 | encode));
12893 }
12894
12895 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12896 NOT_LP64(assert(VM_Version::supports_sse(), ""));
12897 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12898 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12899 emit_int16(0x2C, (0xC0 | encode));
12900 }
12901
12902 void Assembler::decl(Register dst) {
12903 // Don't use it directly. Use MacroAssembler::decrementl() instead.
12904 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12905 int encode = prefix_and_encode(dst->encoding());
12906 emit_int16((unsigned char)0xFF, (0xC8 | encode));
12907 }
12908
12909 void Assembler::decq(Register dst) {
12910 // Don't use it directly. Use MacroAssembler::decrementq() instead.
12911 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12912 int encode = prefixq_and_encode(dst->encoding());
12913 emit_int16((unsigned char)0xFF, 0xC8 | encode);
12914 }
12915
12916 void Assembler::decq(Address dst) {
12917 // Don't use it directly. Use MacroAssembler::decrementq() instead.
12918 InstructionMark im(this);
12919 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12920 emit_operand(rcx, dst, 0);
12921 }
12922
12923 void Assembler::fxrstor(Address src) {
12924 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12925 emit_operand(as_Register(1), src, 0);
12926 }
12927
12928 void Assembler::xrstor(Address src) {
12929 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12930 emit_operand(as_Register(5), src, 0);
12931 }
12932
12933 void Assembler::fxsave(Address dst) {
12934 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12935 emit_operand(as_Register(0), dst, 0);
12936 }
12937
12938 void Assembler::xsave(Address dst) {
12939 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12940 emit_operand(as_Register(4), dst, 0);
12941 }
12942
12943 void Assembler::idivq(Register src) {
12944 int encode = prefixq_and_encode(src->encoding());
12945 emit_int16((unsigned char)0xF7, (0xF8 | encode));
12946 }
12947
12948 void Assembler::divq(Register src) {
12949 int encode = prefixq_and_encode(src->encoding());
12950 emit_int16((unsigned char)0xF7, (0xF0 | encode));
12951 }
12952
12953 void Assembler::imulq(Register dst, Register src) {
12954 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12955 emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12956 }
12957
12958 void Assembler::imulq(Register src) {
12959 int encode = prefixq_and_encode(src->encoding());
12960 emit_int16((unsigned char)0xF7, (0xE8 | encode));
12961 }
12962
12963 void Assembler::imulq(Register dst, Address src, int32_t value) {
12964 InstructionMark im(this);
12965 prefixq(src, dst);
12966 if (is8bit(value)) {
12967 emit_int8((unsigned char)0x6B);
12968 emit_operand(dst, src, 1);
12969 emit_int8(value);
12970 } else {
12971 emit_int8((unsigned char)0x69);
12972 emit_operand(dst, src, 4);
12973 emit_int32(value);
12974 }
12975 }
12976
12977 void Assembler::imulq(Register dst, Register src, int value) {
12978 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12979 if (is8bit(value)) {
12980 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12981 } else {
12982 emit_int16(0x69, (0xC0 | encode));
12983 emit_int32(value);
12984 }
12985 }
12986
12987 void Assembler::imulq(Register dst, Address src) {
12988 InstructionMark im(this);
12989 emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12990 emit_operand(dst, src, 0);
12991 }
12992
12993 void Assembler::incl(Register dst) {
12994 // Don't use it directly. Use MacroAssembler::incrementl() instead.
12995 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12996 int encode = prefix_and_encode(dst->encoding());
12997 emit_int16((unsigned char)0xFF, (0xC0 | encode));
12998 }
12999
13000 void Assembler::incq(Register dst) {
13001 // Don't use it directly. Use MacroAssembler::incrementq() instead.
13002 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13003 int encode = prefixq_and_encode(dst->encoding());
13004 emit_int16((unsigned char)0xFF, (0xC0 | encode));
13005 }
13006
13007 void Assembler::incq(Address dst) {
13008 // Don't use it directly. Use MacroAssembler::incrementq() instead.
13009 InstructionMark im(this);
13010 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13011 emit_operand(rax, dst, 0);
13012 }
13013
13014 void Assembler::lea(Register dst, Address src) {
13015 leaq(dst, src);
13016 }
13017
13018 void Assembler::leaq(Register dst, Address src) {
13019 InstructionMark im(this);
13020 emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13021 emit_operand(dst, src, 0);
13022 }
13023
13024 void Assembler::mov64(Register dst, int64_t imm64) {
13025 InstructionMark im(this);
13026 int encode = prefixq_and_encode(dst->encoding());
13027 emit_int8(0xB8 | encode);
13028 emit_int64(imm64);
13029 }
13030
13031 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13032 InstructionMark im(this);
13033 int encode = prefixq_and_encode(dst->encoding());
13034 emit_int8(0xB8 | encode);
13035 emit_data64(imm64, rtype, format);
13036 }
13037
13038 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13039 InstructionMark im(this);
13040 int encode = prefixq_and_encode(dst->encoding());
13041 emit_int8(0xB8 | encode);
13042 emit_data64(imm64, rspec);
13043 }
13044
13045 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13046 InstructionMark im(this);
13047 int encode = prefix_and_encode(dst->encoding());
13048 emit_int8(0xB8 | encode);
13049 emit_data((int)imm32, rspec, narrow_oop_operand);
13050 }
13051
13052 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
13053 InstructionMark im(this);
13054 prefix(dst);
13055 emit_int8((unsigned char)0xC7);
13056 emit_operand(rax, dst, 4);
13057 emit_data((int)imm32, rspec, narrow_oop_operand);
13058 }
13059
13060 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13061 InstructionMark im(this);
13062 int encode = prefix_and_encode(src1->encoding());
13063 emit_int16((unsigned char)0x81, (0xF8 | encode));
13064 emit_data((int)imm32, rspec, narrow_oop_operand);
13065 }
13066
13067 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13068 InstructionMark im(this);
13069 prefix(src1);
13070 emit_int8((unsigned char)0x81);
13071 emit_operand(rax, src1, 4);
13072 emit_data((int)imm32, rspec, narrow_oop_operand);
13073 }
13074
13075 void Assembler::lzcntq(Register dst, Register src) {
13076 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13077 emit_int8((unsigned char)0xF3);
13078 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13079 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13080 }
13081
13082 void Assembler::lzcntq(Register dst, Address src) {
13083 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13084 InstructionMark im(this);
13085 emit_int8((unsigned char)0xF3);
13086 prefixq(src, dst);
13087 emit_int16(0x0F, (unsigned char)0xBD);
13088 emit_operand(dst, src, 0);
13089 }
13090
13091 void Assembler::movdq(XMMRegister dst, Register src) {
13092 // table D-1 says MMX/SSE2
13093 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13094 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13095 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13096 emit_int16(0x6E, (0xC0 | encode));
13097 }
13098
13099 void Assembler::movdq(Register dst, XMMRegister src) {
13100 // table D-1 says MMX/SSE2
13101 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13102 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13103 // swap src/dst to get correct prefix
13104 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13105 emit_int16(0x7E,
13106 (0xC0 | encode));
13107 }
13108
13109 void Assembler::movq(Register dst, Register src) {
13110 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13111 emit_int16((unsigned char)0x8B,
13112 (0xC0 | encode));
13113 }
13114
13115 void Assembler::movq(Register dst, Address src) {
13116 InstructionMark im(this);
13117 emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13118 emit_operand(dst, src, 0);
13119 }
13120
13121 void Assembler::movq(Address dst, Register src) {
13122 InstructionMark im(this);
13123 emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13124 emit_operand(src, dst, 0);
13125 }
13126
13127 void Assembler::movq(Address dst, int32_t imm32) {
13128 InstructionMark im(this);
13129 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13130 emit_operand(as_Register(0), dst, 4);
13131 emit_int32(imm32);
13132 }
13133
13134 void Assembler::movq(Register dst, int32_t imm32) {
13135 int encode = prefixq_and_encode(dst->encoding());
13136 emit_int16((unsigned char)0xC7, (0xC0 | encode));
13137 emit_int32(imm32);
13138 }
13139
13140 void Assembler::movsbq(Register dst, Address src) {
13141 InstructionMark im(this);
13142 emit_int24(get_prefixq(src, dst),
13143 0x0F,
13144 (unsigned char)0xBE);
13145 emit_operand(dst, src, 0);
13146 }
13147
13148 void Assembler::movsbq(Register dst, Register src) {
13149 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13150 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13151 }
13152
13153 void Assembler::movslq(Register dst, int32_t imm32) {
13154 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
13155 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
13156 // as a result we shouldn't use until tested at runtime...
13157 ShouldNotReachHere();
13158 InstructionMark im(this);
13159 int encode = prefixq_and_encode(dst->encoding());
13160 emit_int8(0xC7 | encode);
13161 emit_int32(imm32);
13162 }
13163
13164 void Assembler::movslq(Address dst, int32_t imm32) {
13165 assert(is_simm32(imm32), "lost bits");
13166 InstructionMark im(this);
13167 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13168 emit_operand(rax, dst, 4);
13169 emit_int32(imm32);
13170 }
13171
13172 void Assembler::movslq(Register dst, Address src) {
13173 InstructionMark im(this);
13174 emit_int16(get_prefixq(src, dst), 0x63);
13175 emit_operand(dst, src, 0);
13176 }
13177
13178 void Assembler::movslq(Register dst, Register src) {
13179 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13180 emit_int16(0x63, (0xC0 | encode));
13181 }
13182
13183 void Assembler::movswq(Register dst, Address src) {
13184 InstructionMark im(this);
13185 emit_int24(get_prefixq(src, dst),
13186 0x0F,
13187 (unsigned char)0xBF);
13188 emit_operand(dst, src, 0);
13189 }
13190
13191 void Assembler::movswq(Register dst, Register src) {
13192 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13193 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13194 }
13195
13196 void Assembler::movzbq(Register dst, Address src) {
13197 InstructionMark im(this);
13198 emit_int24(get_prefixq(src, dst),
13199 0x0F,
13200 (unsigned char)0xB6);
13201 emit_operand(dst, src, 0);
13202 }
13203
13204 void Assembler::movzbq(Register dst, Register src) {
13205 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13206 emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13207 }
13208
13209 void Assembler::movzwq(Register dst, Address src) {
13210 InstructionMark im(this);
13211 emit_int24(get_prefixq(src, dst),
13212 0x0F,
13213 (unsigned char)0xB7);
13214 emit_operand(dst, src, 0);
13215 }
13216
13217 void Assembler::movzwq(Register dst, Register src) {
13218 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13219 emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13220 }
13221
13222 void Assembler::mulq(Address src) {
13223 InstructionMark im(this);
13224 emit_int16(get_prefixq(src), (unsigned char)0xF7);
13225 emit_operand(rsp, src, 0);
13226 }
13227
13228 void Assembler::mulq(Register src) {
13229 int encode = prefixq_and_encode(src->encoding());
13230 emit_int16((unsigned char)0xF7, (0xE0 | encode));
13231 }
13232
13233 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13234 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13235 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13236 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13237 emit_int16((unsigned char)0xF6, (0xC0 | encode));
13238 }
13239
13240 void Assembler::negq(Register dst) {
13241 int encode = prefixq_and_encode(dst->encoding());
13242 emit_int16((unsigned char)0xF7, (0xD8 | encode));
13243 }
13244
13245 void Assembler::negq(Address dst) {
13246 InstructionMark im(this);
13247 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13248 emit_operand(as_Register(3), dst, 0);
13249 }
13250
13251 void Assembler::notq(Register dst) {
13252 int encode = prefixq_and_encode(dst->encoding());
13253 emit_int16((unsigned char)0xF7, (0xD0 | encode));
13254 }
13255
13256 void Assembler::btsq(Address dst, int imm8) {
13257 assert(isByte(imm8), "not a byte");
13258 InstructionMark im(this);
13259 emit_int24(get_prefixq(dst),
13260 0x0F,
13261 (unsigned char)0xBA);
13262 emit_operand(rbp /* 5 */, dst, 1);
13263 emit_int8(imm8);
13264 }
13265
13266 void Assembler::btrq(Address dst, int imm8) {
13267 assert(isByte(imm8), "not a byte");
13268 InstructionMark im(this);
13269 emit_int24(get_prefixq(dst),
13270 0x0F,
13271 (unsigned char)0xBA);
13272 emit_operand(rsi /* 6 */, dst, 1);
13273 emit_int8(imm8);
13274 }
13275
13276 void Assembler::orq(Address dst, int32_t imm32) {
13277 InstructionMark im(this);
13278 prefixq(dst);
13279 emit_arith_operand(0x81, as_Register(1), dst, imm32);
13280 }
13281
13282 void Assembler::orq(Address dst, Register src) {
13283 InstructionMark im(this);
13284 emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13285 emit_operand(src, dst, 0);
13286 }
13287
13288 void Assembler::orq(Register dst, int32_t imm32) {
13289 (void) prefixq_and_encode(dst->encoding());
13290 emit_arith(0x81, 0xC8, dst, imm32);
13291 }
13292
13293 void Assembler::orq(Register dst, Address src) {
13294 InstructionMark im(this);
13295 emit_int16(get_prefixq(src, dst), 0x0B);
13296 emit_operand(dst, src, 0);
13297 }
13298
13299 void Assembler::orq(Register dst, Register src) {
13300 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13301 emit_arith(0x0B, 0xC0, dst, src);
13302 }
13303
13304 void Assembler::popcntq(Register dst, Address src) {
13305 assert(VM_Version::supports_popcnt(), "must support");
13306 InstructionMark im(this);
13307 emit_int32((unsigned char)0xF3,
13308 get_prefixq(src, dst),
13309 0x0F,
13310 (unsigned char)0xB8);
13311 emit_operand(dst, src, 0);
13312 }
13313
13314 void Assembler::popcntq(Register dst, Register src) {
13315 assert(VM_Version::supports_popcnt(), "must support");
13316 emit_int8((unsigned char)0xF3);
13317 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13318 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13319 }
13320
13321 void Assembler::popq(Address dst) {
13322 InstructionMark im(this);
13323 emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13324 emit_operand(rax, dst, 0);
13325 }
13326
13327 void Assembler::popq(Register dst) {
13328 emit_int8((unsigned char)0x58 | dst->encoding());
13329 }
13330
13331 // Precomputable: popa, pusha, vzeroupper
13332
13333 // The result of these routines are invariant from one invocation to another
13334 // invocation for the duration of a run. Caching the result on bootstrap
13335 // and copying it out on subsequent invocations can thus be beneficial
13336 static bool precomputed = false;
13337
13338 static u_char* popa_code = NULL;
13339 static int popa_len = 0;
13340
13341 static u_char* pusha_code = NULL;
13342 static int pusha_len = 0;
13343
13344 static u_char* vzup_code = NULL;
13345 static int vzup_len = 0;
13346
13347 void Assembler::precompute_instructions() {
13348 assert(!Universe::is_fully_initialized(), "must still be single threaded");
13349 guarantee(!precomputed, "only once");
13350 precomputed = true;
13351 ResourceMark rm;
13352
13353 // Make a temporary buffer big enough for the routines we're capturing
13354 int size = 256;
13355 char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13356 CodeBuffer buffer((address)tmp_code, size);
13357 MacroAssembler masm(&buffer);
13358
13359 address begin_popa = masm.code_section()->end();
13360 masm.popa_uncached();
13361 address end_popa = masm.code_section()->end();
13362 masm.pusha_uncached();
13363 address end_pusha = masm.code_section()->end();
13364 masm.vzeroupper_uncached();
13365 address end_vzup = masm.code_section()->end();
13366
13367 // Save the instructions to permanent buffers.
13368 popa_len = (int)(end_popa - begin_popa);
13369 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13370 memcpy(popa_code, begin_popa, popa_len);
13371
13372 pusha_len = (int)(end_pusha - end_popa);
13373 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13374 memcpy(pusha_code, end_popa, pusha_len);
13375
13376 vzup_len = (int)(end_vzup - end_pusha);
13377 if (vzup_len > 0) {
13378 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13379 memcpy(vzup_code, end_pusha, vzup_len);
13380 } else {
13381 vzup_code = pusha_code; // dummy
13382 }
13383
13384 assert(masm.code()->total_oop_size() == 0 &&
13385 masm.code()->total_metadata_size() == 0 &&
13386 masm.code()->total_relocation_size() == 0,
13387 "pre-computed code can't reference oops, metadata or contain relocations");
13388 }
13389
13390 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13391 assert(src != NULL, "code to copy must have been pre-computed");
13392 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13393 address end = code_section->end();
13394 memcpy(end, src, src_len);
13395 code_section->set_end(end + src_len);
13396 }
13397
13398 void Assembler::popa() { // 64bit
13399 emit_copy(code_section(), popa_code, popa_len);
13400 }
13401
13402 void Assembler::popa_uncached() { // 64bit
13403 movq(r15, Address(rsp, 0));
13404 movq(r14, Address(rsp, wordSize));
13405 movq(r13, Address(rsp, 2 * wordSize));
13406 movq(r12, Address(rsp, 3 * wordSize));
13407 movq(r11, Address(rsp, 4 * wordSize));
13408 movq(r10, Address(rsp, 5 * wordSize));
13409 movq(r9, Address(rsp, 6 * wordSize));
13410 movq(r8, Address(rsp, 7 * wordSize));
13411 movq(rdi, Address(rsp, 8 * wordSize));
13412 movq(rsi, Address(rsp, 9 * wordSize));
13413 movq(rbp, Address(rsp, 10 * wordSize));
13414 // Skip rsp as it is restored automatically to the value
13415 // before the corresponding pusha when popa is done.
13416 movq(rbx, Address(rsp, 12 * wordSize));
13417 movq(rdx, Address(rsp, 13 * wordSize));
13418 movq(rcx, Address(rsp, 14 * wordSize));
13419 movq(rax, Address(rsp, 15 * wordSize));
13420
13421 addq(rsp, 16 * wordSize);
13422 }
13423
13424 // Does not actually store the value of rsp on the stack.
13425 // The slot for rsp just contains an arbitrary value.
13426 void Assembler::pusha() { // 64bit
13427 emit_copy(code_section(), pusha_code, pusha_len);
13428 }
13429
13430 // Does not actually store the value of rsp on the stack.
13431 // The slot for rsp just contains an arbitrary value.
13432 void Assembler::pusha_uncached() { // 64bit
13433 subq(rsp, 16 * wordSize);
13434
13435 movq(Address(rsp, 15 * wordSize), rax);
13436 movq(Address(rsp, 14 * wordSize), rcx);
13437 movq(Address(rsp, 13 * wordSize), rdx);
13438 movq(Address(rsp, 12 * wordSize), rbx);
13439 // Skip rsp as the value is normally not used. There are a few places where
13440 // the original value of rsp needs to be known but that can be computed
13441 // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13442 movq(Address(rsp, 10 * wordSize), rbp);
13443 movq(Address(rsp, 9 * wordSize), rsi);
13444 movq(Address(rsp, 8 * wordSize), rdi);
13445 movq(Address(rsp, 7 * wordSize), r8);
13446 movq(Address(rsp, 6 * wordSize), r9);
13447 movq(Address(rsp, 5 * wordSize), r10);
13448 movq(Address(rsp, 4 * wordSize), r11);
13449 movq(Address(rsp, 3 * wordSize), r12);
13450 movq(Address(rsp, 2 * wordSize), r13);
13451 movq(Address(rsp, wordSize), r14);
13452 movq(Address(rsp, 0), r15);
13453 }
13454
13455 void Assembler::vzeroupper() {
13456 emit_copy(code_section(), vzup_code, vzup_len);
13457 }
13458
13459 void Assembler::pushq(Address src) {
13460 InstructionMark im(this);
13461 emit_int16(get_prefixq(src), (unsigned char)0xFF);
13462 emit_operand(rsi, src, 0);
13463 }
13464
13465 void Assembler::rclq(Register dst, int imm8) {
13466 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13467 int encode = prefixq_and_encode(dst->encoding());
13468 if (imm8 == 1) {
13469 emit_int16((unsigned char)0xD1, (0xD0 | encode));
13470 } else {
13471 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13472 }
13473 }
13474
13475 void Assembler::rcrq(Register dst, int imm8) {
13476 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13477 int encode = prefixq_and_encode(dst->encoding());
13478 if (imm8 == 1) {
13479 emit_int16((unsigned char)0xD1, (0xD8 | encode));
13480 } else {
13481 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13482 }
13483 }
13484
13485 void Assembler::rorxl(Register dst, Register src, int imm8) {
13486 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13487 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13488 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13489 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13490 }
13491
13492 void Assembler::rorxl(Register dst, Address src, int imm8) {
13493 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13494 InstructionMark im(this);
13495 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13496 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13497 emit_int8((unsigned char)0xF0);
13498 emit_operand(dst, src, 1);
13499 emit_int8(imm8);
13500 }
13501
13502 void Assembler::rorxq(Register dst, Register src, int imm8) {
13503 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13504 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13505 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13506 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13507 }
13508
13509 void Assembler::rorxq(Register dst, Address src, int imm8) {
13510 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13511 InstructionMark im(this);
13512 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13513 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13514 emit_int8((unsigned char)0xF0);
13515 emit_operand(dst, src, 1);
13516 emit_int8(imm8);
13517 }
13518
13519 #ifdef _LP64
13520 void Assembler::salq(Address dst, int imm8) {
13521 InstructionMark im(this);
13522 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13523 if (imm8 == 1) {
13524 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13525 emit_operand(as_Register(4), dst, 0);
13526 }
13527 else {
13528 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13529 emit_operand(as_Register(4), dst, 1);
13530 emit_int8(imm8);
13531 }
13532 }
13533
13534 void Assembler::salq(Address dst) {
13535 InstructionMark im(this);
13536 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13537 emit_operand(as_Register(4), dst, 0);
13538 }
13539
13540 void Assembler::salq(Register dst, int imm8) {
13541 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13542 int encode = prefixq_and_encode(dst->encoding());
13543 if (imm8 == 1) {
13544 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13545 } else {
13546 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13547 }
13548 }
13549
13550 void Assembler::salq(Register dst) {
13551 int encode = prefixq_and_encode(dst->encoding());
13552 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13553 }
13554
13555 void Assembler::sarq(Address dst, int imm8) {
13556 InstructionMark im(this);
13557 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13558 if (imm8 == 1) {
13559 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13560 emit_operand(as_Register(7), dst, 0);
13561 }
13562 else {
13563 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13564 emit_operand(as_Register(7), dst, 1);
13565 emit_int8(imm8);
13566 }
13567 }
13568
13569 void Assembler::sarq(Address dst) {
13570 InstructionMark im(this);
13571 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13572 emit_operand(as_Register(7), dst, 0);
13573 }
13574
13575 void Assembler::sarq(Register dst, int imm8) {
13576 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13577 int encode = prefixq_and_encode(dst->encoding());
13578 if (imm8 == 1) {
13579 emit_int16((unsigned char)0xD1, (0xF8 | encode));
13580 } else {
13581 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13582 }
13583 }
13584
13585 void Assembler::sarq(Register dst) {
13586 int encode = prefixq_and_encode(dst->encoding());
13587 emit_int16((unsigned char)0xD3, (0xF8 | encode));
13588 }
13589 #endif
13590
13591 void Assembler::sbbq(Address dst, int32_t imm32) {
13592 InstructionMark im(this);
13593 prefixq(dst);
13594 emit_arith_operand(0x81, rbx, dst, imm32);
13595 }
13596
13597 void Assembler::sbbq(Register dst, int32_t imm32) {
13598 (void) prefixq_and_encode(dst->encoding());
13599 emit_arith(0x81, 0xD8, dst, imm32);
13600 }
13601
13602 void Assembler::sbbq(Register dst, Address src) {
13603 InstructionMark im(this);
13604 emit_int16(get_prefixq(src, dst), 0x1B);
13605 emit_operand(dst, src, 0);
13606 }
13607
13608 void Assembler::sbbq(Register dst, Register src) {
13609 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13610 emit_arith(0x1B, 0xC0, dst, src);
13611 }
13612
13613 void Assembler::shlq(Register dst, int imm8) {
13614 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13615 int encode = prefixq_and_encode(dst->encoding());
13616 if (imm8 == 1) {
13617 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13618 } else {
13619 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13620 }
13621 }
13622
13623 void Assembler::shlq(Register dst) {
13624 int encode = prefixq_and_encode(dst->encoding());
13625 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13626 }
13627
13628 void Assembler::shrq(Register dst, int imm8) {
13629 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13630 int encode = prefixq_and_encode(dst->encoding());
13631 if (imm8 == 1) {
13632 emit_int16((unsigned char)0xD1, (0xE8 | encode));
13633 }
13634 else {
13635 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13636 }
13637 }
13638
13639 void Assembler::shrq(Register dst) {
13640 int encode = prefixq_and_encode(dst->encoding());
13641 emit_int16((unsigned char)0xD3, 0xE8 | encode);
13642 }
13643
13644 void Assembler::shrq(Address dst) {
13645 InstructionMark im(this);
13646 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13647 emit_operand(as_Register(5), dst, 0);
13648 }
13649
13650 void Assembler::shrq(Address dst, int imm8) {
13651 InstructionMark im(this);
13652 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13653 if (imm8 == 1) {
13654 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13655 emit_operand(as_Register(5), dst, 0);
13656 }
13657 else {
13658 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13659 emit_operand(as_Register(5), dst, 1);
13660 emit_int8(imm8);
13661 }
13662 }
13663
13664 void Assembler::subq(Address dst, int32_t imm32) {
13665 InstructionMark im(this);
13666 prefixq(dst);
13667 emit_arith_operand(0x81, rbp, dst, imm32);
13668 }
13669
13670 void Assembler::subq(Address dst, Register src) {
13671 InstructionMark im(this);
13672 emit_int16(get_prefixq(dst, src), 0x29);
13673 emit_operand(src, dst, 0);
13674 }
13675
13676 void Assembler::subq(Register dst, int32_t imm32) {
13677 (void) prefixq_and_encode(dst->encoding());
13678 emit_arith(0x81, 0xE8, dst, imm32);
13679 }
13680
13681 // Force generation of a 4 byte immediate value even if it fits into 8bit
13682 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13683 (void) prefixq_and_encode(dst->encoding());
13684 emit_arith_imm32(0x81, 0xE8, dst, imm32);
13685 }
13686
13687 void Assembler::subq(Register dst, Address src) {
13688 InstructionMark im(this);
13689 emit_int16(get_prefixq(src, dst), 0x2B);
13690 emit_operand(dst, src, 0);
13691 }
13692
13693 void Assembler::subq(Register dst, Register src) {
13694 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13695 emit_arith(0x2B, 0xC0, dst, src);
13696 }
13697
13698 void Assembler::testq(Address dst, int32_t imm32) {
13699 InstructionMark im(this);
13700 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13701 emit_operand(as_Register(0), dst, 4);
13702 emit_int32(imm32);
13703 }
13704
13705 void Assembler::testq(Register dst, int32_t imm32) {
13706 // not using emit_arith because test
13707 // doesn't support sign-extension of
13708 // 8bit operands
13709 if (dst == rax) {
13710 prefix(REX_W);
13711 emit_int8((unsigned char)0xA9);
13712 emit_int32(imm32);
13713 } else {
13714 int encode = dst->encoding();
13715 encode = prefixq_and_encode(encode);
13716 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13717 emit_int32(imm32);
13718 }
13719 }
13720
13721 void Assembler::testq(Register dst, Register src) {
13722 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13723 emit_arith(0x85, 0xC0, dst, src);
13724 }
13725
13726 void Assembler::testq(Register dst, Address src) {
13727 InstructionMark im(this);
13728 emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13729 emit_operand(dst, src, 0);
13730 }
13731
13732 void Assembler::xaddq(Address dst, Register src) {
13733 InstructionMark im(this);
13734 emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13735 emit_operand(src, dst, 0);
13736 }
13737
13738 void Assembler::xchgq(Register dst, Address src) {
13739 InstructionMark im(this);
13740 emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13741 emit_operand(dst, src, 0);
13742 }
13743
13744 void Assembler::xchgq(Register dst, Register src) {
13745 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13746 emit_int16((unsigned char)0x87, (0xc0 | encode));
13747 }
13748
13749 void Assembler::xorq(Register dst, Register src) {
13750 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13751 emit_arith(0x33, 0xC0, dst, src);
13752 }
13753
13754 void Assembler::xorq(Register dst, Address src) {
13755 InstructionMark im(this);
13756 emit_int16(get_prefixq(src, dst), 0x33);
13757 emit_operand(dst, src, 0);
13758 }
13759
13760 void Assembler::xorq(Register dst, int32_t imm32) {
13761 (void) prefixq_and_encode(dst->encoding());
13762 emit_arith(0x81, 0xF0, dst, imm32);
13763 }
13764
13765 void Assembler::xorq(Address dst, int32_t imm32) {
13766 InstructionMark im(this);
13767 prefixq(dst);
13768 emit_arith_operand(0x81, as_Register(6), dst, imm32);
13769 }
13770
13771 void Assembler::xorq(Address dst, Register src) {
13772 InstructionMark im(this);
13773 emit_int16(get_prefixq(dst, src), 0x31);
13774 emit_operand(src, dst, 0);
13775 }
13776
13777 #endif // !LP64
13778
13779 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13780 if (VM_Version::supports_evex()) {
13781 _tuple_type = tuple_type;
13782 _input_size_in_bits = input_size_in_bits;
13783 }
13784 }