1 /*
2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableBarrierSet.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/objectMonitor.hpp"
34 #include "runtime/os.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37 #include "runtime/vm_version.hpp"
38 #include "utilities/macros.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
49 // Implementation of AddressLiteral
50
51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
53 // -----------------Table 4.5 -------------------- //
54 16, 32, 64, // EVEX_FV(0)
55 4, 4, 4, // EVEX_FV(1) - with Evex.b
56 16, 32, 64, // EVEX_FV(2) - with Evex.w
57 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
58 8, 16, 32, // EVEX_HV(0)
59 4, 4, 4, // EVEX_HV(1) - with Evex.b
60 // -----------------Table 4.6 -------------------- //
61 16, 32, 64, // EVEX_FVM(0)
62 1, 1, 1, // EVEX_T1S(0)
63 2, 2, 2, // EVEX_T1S(1)
64 4, 4, 4, // EVEX_T1S(2)
65 8, 8, 8, // EVEX_T1S(3)
66 4, 4, 4, // EVEX_T1F(0)
67 8, 8, 8, // EVEX_T1F(1)
68 8, 8, 8, // EVEX_T2(0)
69 0, 16, 16, // EVEX_T2(1)
70 0, 16, 16, // EVEX_T4(0)
71 0, 0, 32, // EVEX_T4(1)
72 0, 0, 32, // EVEX_T8(0)
73 8, 16, 32, // EVEX_HVM(0)
74 4, 8, 16, // EVEX_QVM(0)
75 2, 4, 8, // EVEX_OVM(0)
76 16, 16, 16, // EVEX_M128(0)
77 8, 32, 64, // EVEX_DUP(0)
78 0, 0, 0 // EVEX_NTUP
79 };
80
81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
82 _is_lval = false;
83 _target = target;
84 switch (rtype) {
85 case relocInfo::oop_type:
86 case relocInfo::metadata_type:
87 // Oops are a special case. Normally they would be their own section
88 // but in cases like icBuffer they are literals in the code stream that
89 // we don't have a section for. We use none so that we get a literal address
90 // which is always patchable.
91 break;
92 case relocInfo::external_word_type:
93 _rspec = external_word_Relocation::spec(target);
94 break;
95 case relocInfo::internal_word_type:
96 _rspec = internal_word_Relocation::spec(target);
97 break;
98 case relocInfo::opt_virtual_call_type:
99 _rspec = opt_virtual_call_Relocation::spec();
100 break;
101 case relocInfo::static_call_type:
102 _rspec = static_call_Relocation::spec();
103 break;
104 case relocInfo::runtime_call_type:
105 _rspec = runtime_call_Relocation::spec();
106 break;
107 case relocInfo::poll_type:
108 case relocInfo::poll_return_type:
109 _rspec = Relocation::spec_simple(rtype);
110 break;
111 case relocInfo::none:
112 break;
113 default:
114 ShouldNotReachHere();
115 break;
116 }
117 }
118
119 // Implementation of Address
120
121 #ifdef _LP64
122
123 Address Address::make_array(ArrayAddress adr) {
124 // Not implementable on 64bit machines
125 // Should have been handled higher up the call chain.
126 ShouldNotReachHere();
127 return Address();
128 }
129
130 // exceedingly dangerous constructor
131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
132 _base = noreg;
133 _index = noreg;
134 _scale = no_scale;
135 _disp = disp;
136 _xmmindex = xnoreg;
137 _isxmmindex = false;
138 switch (rtype) {
139 case relocInfo::external_word_type:
140 _rspec = external_word_Relocation::spec(loc);
141 break;
142 case relocInfo::internal_word_type:
143 _rspec = internal_word_Relocation::spec(loc);
144 break;
145 case relocInfo::runtime_call_type:
146 // HMM
147 _rspec = runtime_call_Relocation::spec();
148 break;
149 case relocInfo::poll_type:
150 case relocInfo::poll_return_type:
151 _rspec = Relocation::spec_simple(rtype);
152 break;
153 case relocInfo::none:
154 break;
155 default:
156 ShouldNotReachHere();
157 }
158 }
159 #else // LP64
160
161 Address Address::make_array(ArrayAddress adr) {
162 AddressLiteral base = adr.base();
163 Address index = adr.index();
164 assert(index._disp == 0, "must not have disp"); // maybe it can?
165 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
166 array._rspec = base._rspec;
167 return array;
168 }
169
170 // exceedingly dangerous constructor
171 Address::Address(address loc, RelocationHolder spec) {
172 _base = noreg;
173 _index = noreg;
174 _scale = no_scale;
175 _disp = (intptr_t) loc;
176 _rspec = spec;
177 _xmmindex = xnoreg;
178 _isxmmindex = false;
179 }
180
181 #endif // _LP64
182
183
184
185 // Convert the raw encoding form into the form expected by the constructor for
186 // Address. An index of 4 (rsp) corresponds to having no index, so convert
187 // that to noreg for the Address constructor.
188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
189 RelocationHolder rspec = RelocationHolder::none;
190 if (disp_reloc != relocInfo::none) {
191 rspec = Relocation::spec_simple(disp_reloc);
192 }
193 bool valid_index = index != rsp->encoding();
194 if (valid_index) {
195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
196 madr._rspec = rspec;
197 return madr;
198 } else {
199 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
200 madr._rspec = rspec;
201 return madr;
202 }
203 }
204
205 // Implementation of Assembler
206
207 int AbstractAssembler::code_fill_byte() {
208 return (u_char)'\xF4'; // hlt
209 }
210
211 void Assembler::init_attributes(void) {
212 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
213 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
214 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
215 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
216 NOT_LP64(_is_managed = false;)
217 _attributes = NULL;
218 }
219
220
221 void Assembler::membar(Membar_mask_bits order_constraint) {
222 // We only have to handle StoreLoad
223 if (order_constraint & StoreLoad) {
224 // All usable chips support "locked" instructions which suffice
225 // as barriers, and are much faster than the alternative of
226 // using cpuid instruction. We use here a locked add [esp-C],0.
227 // This is conveniently otherwise a no-op except for blowing
228 // flags, and introducing a false dependency on target memory
229 // location. We can't do anything with flags, but we can avoid
230 // memory dependencies in the current method by locked-adding
231 // somewhere else on the stack. Doing [esp+C] will collide with
232 // something on stack in current method, hence we go for [esp-C].
233 // It is convenient since it is almost always in data cache, for
234 // any small C. We need to step back from SP to avoid data
235 // dependencies with other things on below SP (callee-saves, for
236 // example). Without a clear way to figure out the minimal safe
237 // distance from SP, it makes sense to step back the complete
238 // cache line, as this will also avoid possible second-order effects
239 // with locked ops against the cache line. Our choice of offset
240 // is bounded by x86 operand encoding, which should stay within
241 // [-128; +127] to have the 8-byte displacement encoding.
242 //
243 // Any change to this code may need to revisit other places in
244 // the code where this idiom is used, in particular the
245 // orderAccess code.
246
247 int offset = -VM_Version::L1_line_size();
248 if (offset < -128) {
249 offset = -128;
250 }
251
252 lock();
253 addl(Address(rsp, offset), 0);// Assert the lock# signal here
254 }
255 }
256
257 // make this go away someday
258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
259 if (rtype == relocInfo::none)
260 emit_int32(data);
261 else
262 emit_data(data, Relocation::spec_simple(rtype), format);
263 }
264
265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
266 assert(imm_operand == 0, "default format must be immediate in this file");
267 assert(inst_mark() != NULL, "must be inside InstructionMark");
268 if (rspec.type() != relocInfo::none) {
269 #ifdef ASSERT
270 check_relocation(rspec, format);
271 #endif
272 // Do not use AbstractAssembler::relocate, which is not intended for
273 // embedded words. Instead, relocate to the enclosing instruction.
274
275 // hack. call32 is too wide for mask so use disp32
276 if (format == call32_operand)
277 code_section()->relocate(inst_mark(), rspec, disp32_operand);
278 else
279 code_section()->relocate(inst_mark(), rspec, format);
280 }
281 emit_int32(data);
282 }
283
284 static int encode(Register r) {
285 int enc = r->encoding();
286 if (enc >= 8) {
287 enc -= 8;
288 }
289 return enc;
290 }
291
292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
293 assert(dst->has_byte_register(), "must have byte register");
294 assert(isByte(op1) && isByte(op2), "wrong opcode");
295 assert(isByte(imm8), "not a byte");
296 assert((op1 & 0x01) == 0, "should be 8bit operation");
297 emit_int24(op1, (op2 | encode(dst)), imm8);
298 }
299
300
301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
302 assert(isByte(op1) && isByte(op2), "wrong opcode");
303 assert(op1 == 0x81, "Unexpected opcode");
304 if (is8bit(imm32)) {
305 emit_int24(op1 | 0x02, // set sign bit
306 op2 | encode(dst),
307 imm32 & 0xFF);
308 } else if (dst == rax) {
309 switch (op2) {
310 case 0xD0: emit_int8(0x15); break; // adc
311 case 0xC0: emit_int8(0x05); break; // add
312 case 0xE0: emit_int8(0x25); break; // and
313 case 0xF8: emit_int8(0x3D); break; // cmp
314 case 0xC8: emit_int8(0x0D); break; // or
315 case 0xD8: emit_int8(0x1D); break; // sbb
316 case 0xE8: emit_int8(0x2D); break; // sub
317 case 0xF0: emit_int8(0x35); break; // xor
318 default: ShouldNotReachHere();
319 }
320 emit_int32(imm32);
321 } else {
322 emit_int16(op1, (op2 | encode(dst)));
323 emit_int32(imm32);
324 }
325 }
326
327 // Force generation of a 4 byte immediate value even if it fits into 8bit
328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
329 assert(isByte(op1) && isByte(op2), "wrong opcode");
330 assert((op1 & 0x01) == 1, "should be 32bit operation");
331 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
332 emit_int16(op1, (op2 | encode(dst)));
333 emit_int32(imm32);
334 }
335
336 // immediate-to-memory forms
337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
338 assert((op1 & 0x01) == 1, "should be 32bit operation");
339 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
340 if (is8bit(imm32)) {
341 emit_int8(op1 | 0x02); // set sign bit
342 emit_operand(rm, adr, 1);
343 emit_int8(imm32 & 0xFF);
344 } else {
345 emit_int8(op1);
346 emit_operand(rm, adr, 4);
347 emit_int32(imm32);
348 }
349 }
350
351 void Assembler::emit_arith_operand_imm32(int op1, Register rm, Address adr, int32_t imm32) {
352 assert(op1 == 0x81, "unexpected opcode");
353 emit_int8(op1);
354 emit_operand(rm, adr, 4);
355 emit_int32(imm32);
356 }
357
358 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
359 assert(isByte(op1) && isByte(op2), "wrong opcode");
360 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
361 }
362
363
364 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
365 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
366 int mod_idx = 0;
367 // We will test if the displacement fits the compressed format and if so
368 // apply the compression to the displacement iff the result is8bit.
369 if (VM_Version::supports_evex() && is_evex_inst) {
370 switch (cur_tuple_type) {
371 case EVEX_FV:
372 if ((cur_encoding & VEX_W) == VEX_W) {
373 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
374 } else {
375 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
376 }
377 break;
378
379 case EVEX_HV:
380 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
381 break;
382
383 case EVEX_FVM:
384 break;
385
386 case EVEX_T1S:
387 switch (in_size_in_bits) {
388 case EVEX_8bit:
389 break;
390
391 case EVEX_16bit:
392 mod_idx = 1;
393 break;
394
395 case EVEX_32bit:
396 mod_idx = 2;
397 break;
398
399 case EVEX_64bit:
400 mod_idx = 3;
401 break;
402 }
403 break;
404
405 case EVEX_T1F:
406 case EVEX_T2:
407 case EVEX_T4:
408 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
409 break;
410
411 case EVEX_T8:
412 break;
413
414 case EVEX_HVM:
415 break;
416
417 case EVEX_QVM:
418 break;
419
420 case EVEX_OVM:
421 break;
422
423 case EVEX_M128:
424 break;
425
426 case EVEX_DUP:
427 break;
428
429 default:
430 assert(0, "no valid evex tuple_table entry");
431 break;
432 }
433
434 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
435 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
436 if ((disp % disp_factor) == 0) {
437 int new_disp = disp / disp_factor;
438 if ((-0x80 <= new_disp && new_disp < 0x80)) {
439 disp = new_disp;
440 }
441 } else {
442 return false;
443 }
444 }
445 }
446 return (-0x80 <= disp && disp < 0x80);
447 }
448
449
450 bool Assembler::emit_compressed_disp_byte(int &disp) {
451 int mod_idx = 0;
452 // We will test if the displacement fits the compressed format and if so
453 // apply the compression to the displacement iff the result is8bit.
454 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
455 int evex_encoding = _attributes->get_evex_encoding();
456 int tuple_type = _attributes->get_tuple_type();
457 switch (tuple_type) {
458 case EVEX_FV:
459 if ((evex_encoding & VEX_W) == VEX_W) {
460 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
461 } else {
462 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
463 }
464 break;
465
466 case EVEX_HV:
467 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
468 break;
469
470 case EVEX_FVM:
471 break;
472
473 case EVEX_T1S:
474 switch (_attributes->get_input_size()) {
475 case EVEX_8bit:
476 break;
477
478 case EVEX_16bit:
479 mod_idx = 1;
480 break;
481
482 case EVEX_32bit:
483 mod_idx = 2;
484 break;
485
486 case EVEX_64bit:
487 mod_idx = 3;
488 break;
489 }
490 break;
491
492 case EVEX_T1F:
493 case EVEX_T2:
494 case EVEX_T4:
495 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
496 break;
497
498 case EVEX_T8:
499 break;
500
501 case EVEX_HVM:
502 break;
503
504 case EVEX_QVM:
505 break;
506
507 case EVEX_OVM:
508 break;
509
510 case EVEX_M128:
511 break;
512
513 case EVEX_DUP:
514 break;
515
516 default:
517 assert(0, "no valid evex tuple_table entry");
518 break;
519 }
520
521 int vector_len = _attributes->get_vector_len();
522 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
523 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
524 if ((disp % disp_factor) == 0) {
525 int new_disp = disp / disp_factor;
526 if (is8bit(new_disp)) {
527 disp = new_disp;
528 }
529 } else {
530 return false;
531 }
532 }
533 }
534 return is8bit(disp);
535 }
536
537 static bool is_valid_encoding(int reg_enc) {
538 return reg_enc >= 0;
539 }
540
541 static int raw_encode(Register reg) {
542 assert(reg == noreg || reg->is_valid(), "sanity");
543 int reg_enc = reg->raw_encoding();
544 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
545 return reg_enc;
546 }
547
548 static int raw_encode(XMMRegister xmmreg) {
549 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
550 int xmmreg_enc = xmmreg->raw_encoding();
551 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
552 return xmmreg_enc;
553 }
554
555 static int raw_encode(KRegister kreg) {
556 assert(kreg == knoreg || kreg->is_valid(), "sanity");
557 int kreg_enc = kreg->raw_encoding();
558 assert(kreg_enc == -1 || is_valid_encoding(kreg_enc), "sanity");
559 return kreg_enc;
560 }
561
562 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
563 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
564 }
565
566 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
567 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
568 }
569
570 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
571 assert((mod & 3) != 0b11, "forbidden");
572 int modrm = modrm_encoding(mod, dst_enc, src_enc);
573 emit_int8(modrm);
574 }
575
576 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
577 int disp) {
578 int modrm = modrm_encoding(mod, dst_enc, src_enc);
579 emit_int16(modrm, disp & 0xFF);
580 }
581
582 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
583 Address::ScaleFactor scale, int index_enc, int base_enc) {
584 int modrm = modrm_encoding(mod, dst_enc, src_enc);
585 int sib = sib_encoding(scale, index_enc, base_enc);
586 emit_int16(modrm, sib);
587 }
588
589 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
590 Address::ScaleFactor scale, int index_enc, int base_enc,
591 int disp) {
592 int modrm = modrm_encoding(mod, dst_enc, src_enc);
593 int sib = sib_encoding(scale, index_enc, base_enc);
594 emit_int24(modrm, sib, disp & 0xFF);
595 }
596
597 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
598 Address::ScaleFactor scale, int disp,
599 RelocationHolder const& rspec,
600 int post_addr_length) {
601 bool no_relocation = (rspec.type() == relocInfo::none);
602
603 if (is_valid_encoding(base_enc)) {
604 if (is_valid_encoding(index_enc)) {
605 assert(scale != Address::no_scale, "inconsistent address");
606 // [base + index*scale + disp]
607 if (disp == 0 && no_relocation &&
608 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
609 // [base + index*scale]
610 // [00 reg 100][ss index base]
611 emit_modrm_sib(0b00, reg_enc, 0b100,
612 scale, index_enc, base_enc);
613 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
614 // [base + index*scale + imm8]
615 // [01 reg 100][ss index base] imm8
616 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
617 scale, index_enc, base_enc,
618 disp);
619 } else {
620 // [base + index*scale + disp32]
621 // [10 reg 100][ss index base] disp32
622 emit_modrm_sib(0b10, reg_enc, 0b100,
623 scale, index_enc, base_enc);
624 emit_data(disp, rspec, disp32_operand);
625 }
626 } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
627 // [rsp + disp]
628 if (disp == 0 && no_relocation) {
629 // [rsp]
630 // [00 reg 100][00 100 100]
631 emit_modrm_sib(0b00, reg_enc, 0b100,
632 Address::times_1, 0b100, 0b100);
633 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
634 // [rsp + imm8]
635 // [01 reg 100][00 100 100] disp8
636 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
637 Address::times_1, 0b100, 0b100,
638 disp);
639 } else {
640 // [rsp + imm32]
641 // [10 reg 100][00 100 100] disp32
642 emit_modrm_sib(0b10, reg_enc, 0b100,
643 Address::times_1, 0b100, 0b100);
644 emit_data(disp, rspec, disp32_operand);
645 }
646 } else {
647 // [base + disp]
648 assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
649 if (disp == 0 && no_relocation &&
650 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
651 // [base]
652 // [00 reg base]
653 emit_modrm(0, reg_enc, base_enc);
654 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
655 // [base + disp8]
656 // [01 reg base] disp8
657 emit_modrm_disp8(0b01, reg_enc, base_enc,
658 disp);
659 } else {
660 // [base + disp32]
661 // [10 reg base] disp32
662 emit_modrm(0b10, reg_enc, base_enc);
663 emit_data(disp, rspec, disp32_operand);
664 }
665 }
666 } else {
667 if (is_valid_encoding(index_enc)) {
668 assert(scale != Address::no_scale, "inconsistent address");
669 // base == noreg
670 // [index*scale + disp]
671 // [00 reg 100][ss index 101] disp32
672 emit_modrm_sib(0b00, reg_enc, 0b100,
673 scale, index_enc, 0b101 /* no base */);
674 emit_data(disp, rspec, disp32_operand);
675 } else if (!no_relocation) {
676 // base == noreg, index == noreg
677 // [disp] (64bit) RIP-RELATIVE (32bit) abs
678 // [00 reg 101] disp32
679
680 emit_modrm(0b00, reg_enc, 0b101 /* no base */);
681 // Note that the RIP-rel. correction applies to the generated
682 // disp field, but _not_ to the target address in the rspec.
683
684 // disp was created by converting the target address minus the pc
685 // at the start of the instruction. That needs more correction here.
686 // intptr_t disp = target - next_ip;
687 assert(inst_mark() != NULL, "must be inside InstructionMark");
688 address next_ip = pc() + sizeof(int32_t) + post_addr_length;
689 int64_t adjusted = disp;
690 // Do rip-rel adjustment for 64bit
691 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
692 assert(is_simm32(adjusted),
693 "must be 32bit offset (RIP relative address)");
694 emit_data((int32_t) adjusted, rspec, disp32_operand);
695
696 } else {
697 // base == noreg, index == noreg, no_relocation == true
698 // 32bit never did this, did everything as the rip-rel/disp code above
699 // [disp] ABSOLUTE
700 // [00 reg 100][00 100 101] disp32
701 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
702 Address::times_1, 0b100, 0b101);
703 emit_data(disp, rspec, disp32_operand);
704 }
705 }
706 }
707
708 void Assembler::emit_operand(Register reg, Register base, Register index,
709 Address::ScaleFactor scale, int disp,
710 RelocationHolder const& rspec,
711 int post_addr_length) {
712 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
713 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
714 scale, disp, rspec, post_addr_length);
715
716 }
717 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
718 Address::ScaleFactor scale, int disp,
719 RelocationHolder const& rspec,
720 int post_addr_length) {
721 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
722 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
723 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
724 scale, disp, rspec, post_addr_length);
725 }
726
727 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
728 Address::ScaleFactor scale, int disp,
729 RelocationHolder const& rspec,
730 int post_addr_length) {
731 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
732 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
733 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
734 scale, disp, rspec, post_addr_length);
735 }
736
737 void Assembler::emit_operand(KRegister kreg, Address adr,
738 int post_addr_length) {
739 emit_operand(kreg, adr._base, adr._index, adr._scale, adr._disp,
740 adr._rspec,
741 post_addr_length);
742 }
743
744 void Assembler::emit_operand(KRegister kreg, Register base, Register index,
745 Address::ScaleFactor scale, int disp,
746 RelocationHolder const& rspec,
747 int post_addr_length) {
748 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
749 emit_operand_helper(raw_encode(kreg), raw_encode(base), raw_encode(index),
750 scale, disp, rspec, post_addr_length);
751 }
752
753 // Secret local extension to Assembler::WhichOperand:
754 #define end_pc_operand (_WhichOperand_limit)
755
756 address Assembler::locate_operand(address inst, WhichOperand which) {
757 // Decode the given instruction, and return the address of
758 // an embedded 32-bit operand word.
759
760 // If "which" is disp32_operand, selects the displacement portion
761 // of an effective address specifier.
762 // If "which" is imm64_operand, selects the trailing immediate constant.
763 // If "which" is call32_operand, selects the displacement of a call or jump.
764 // Caller is responsible for ensuring that there is such an operand,
765 // and that it is 32/64 bits wide.
766
767 // If "which" is end_pc_operand, find the end of the instruction.
768
769 address ip = inst;
770 bool is_64bit = false;
771
772 debug_only(bool has_disp32 = false);
773 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
774
775 again_after_prefix:
776 switch (0xFF & *ip++) {
777
778 // These convenience macros generate groups of "case" labels for the switch.
779 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
780 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
781 case (x)+4: case (x)+5: case (x)+6: case (x)+7
782 #define REP16(x) REP8((x)+0): \
783 case REP8((x)+8)
784
785 case CS_segment:
786 case SS_segment:
787 case DS_segment:
788 case ES_segment:
789 case FS_segment:
790 case GS_segment:
791 // Seems dubious
792 LP64_ONLY(assert(false, "shouldn't have that prefix"));
793 assert(ip == inst+1, "only one prefix allowed");
794 goto again_after_prefix;
795
796 case 0x67:
797 case REX:
798 case REX_B:
799 case REX_X:
800 case REX_XB:
801 case REX_R:
802 case REX_RB:
803 case REX_RX:
804 case REX_RXB:
805 NOT_LP64(assert(false, "64bit prefixes"));
806 goto again_after_prefix;
807
808 case REX_W:
809 case REX_WB:
810 case REX_WX:
811 case REX_WXB:
812 case REX_WR:
813 case REX_WRB:
814 case REX_WRX:
815 case REX_WRXB:
816 NOT_LP64(assert(false, "64bit prefixes"));
817 is_64bit = true;
818 goto again_after_prefix;
819
820 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
821 case 0x88: // movb a, r
822 case 0x89: // movl a, r
823 case 0x8A: // movb r, a
824 case 0x8B: // movl r, a
825 case 0x8F: // popl a
826 debug_only(has_disp32 = true);
827 break;
828
829 case 0x68: // pushq #32
830 if (which == end_pc_operand) {
831 return ip + 4;
832 }
833 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
834 return ip; // not produced by emit_operand
835
836 case 0x66: // movw ... (size prefix)
837 again_after_size_prefix2:
838 switch (0xFF & *ip++) {
839 case REX:
840 case REX_B:
841 case REX_X:
842 case REX_XB:
843 case REX_R:
844 case REX_RB:
845 case REX_RX:
846 case REX_RXB:
847 case REX_W:
848 case REX_WB:
849 case REX_WX:
850 case REX_WXB:
851 case REX_WR:
852 case REX_WRB:
853 case REX_WRX:
854 case REX_WRXB:
855 NOT_LP64(assert(false, "64bit prefix found"));
856 goto again_after_size_prefix2;
857 case 0x8B: // movw r, a
858 case 0x89: // movw a, r
859 debug_only(has_disp32 = true);
860 break;
861 case 0xC7: // movw a, #16
862 debug_only(has_disp32 = true);
863 tail_size = 2; // the imm16
864 break;
865 case 0x0F: // several SSE/SSE2 variants
866 ip--; // reparse the 0x0F
867 goto again_after_prefix;
868 default:
869 ShouldNotReachHere();
870 }
871 break;
872
873 case REP8(0xB8): // movl/q r, #32/#64(oop?)
874 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
875 // these asserts are somewhat nonsensical
876 #ifndef _LP64
877 assert(which == imm_operand || which == disp32_operand,
878 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
879 #else
880 assert((which == call32_operand || which == imm_operand) && is_64bit ||
881 which == narrow_oop_operand && !is_64bit,
882 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
883 #endif // _LP64
884 return ip;
885
886 case 0x69: // imul r, a, #32
887 case 0xC7: // movl a, #32(oop?)
888 tail_size = 4;
889 debug_only(has_disp32 = true); // has both kinds of operands!
890 break;
891
892 case 0x0F: // movx..., etc.
893 switch (0xFF & *ip++) {
894 case 0x3A: // pcmpestri
895 tail_size = 1;
896 case 0x38: // ptest, pmovzxbw
897 ip++; // skip opcode
898 debug_only(has_disp32 = true); // has both kinds of operands!
899 break;
900
901 case 0x70: // pshufd r, r/a, #8
902 debug_only(has_disp32 = true); // has both kinds of operands!
903 case 0x73: // psrldq r, #8
904 tail_size = 1;
905 break;
906
907 case 0x10: // movups
908 case 0x11: // movups
909 case 0x12: // movlps
910 case 0x28: // movaps
911 case 0x2E: // ucomiss
912 case 0x2F: // comiss
913 case 0x54: // andps
914 case 0x55: // andnps
915 case 0x56: // orps
916 case 0x57: // xorps
917 case 0x58: // addpd
918 case 0x59: // mulpd
919 case 0x6E: // movd
920 case 0x7E: // movd
921 case 0x6F: // movdq
922 case 0x7F: // movdq
923 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
924 case 0xFE: // paddd
925 debug_only(has_disp32 = true);
926 break;
927
928 case 0xAD: // shrd r, a, %cl
929 case 0xAF: // imul r, a
930 case 0xBE: // movsbl r, a (movsxb)
931 case 0xBF: // movswl r, a (movsxw)
932 case 0xB6: // movzbl r, a (movzxb)
933 case 0xB7: // movzwl r, a (movzxw)
934 case REP16(0x40): // cmovl cc, r, a
935 case 0xB0: // cmpxchgb
936 case 0xB1: // cmpxchg
937 case 0xC1: // xaddl
938 case 0xC7: // cmpxchg8
939 case REP16(0x90): // setcc a
940 debug_only(has_disp32 = true);
941 // fall out of the switch to decode the address
942 break;
943
944 case 0xC4: // pinsrw r, a, #8
945 debug_only(has_disp32 = true);
946 case 0xC5: // pextrw r, r, #8
947 tail_size = 1; // the imm8
948 break;
949
950 case 0xAC: // shrd r, a, #8
951 debug_only(has_disp32 = true);
952 tail_size = 1; // the imm8
953 break;
954
955 case REP16(0x80): // jcc rdisp32
956 if (which == end_pc_operand) return ip + 4;
957 assert(which == call32_operand, "jcc has no disp32 or imm");
958 return ip;
959 default:
960 ShouldNotReachHere();
961 }
962 break;
963
964 case 0x81: // addl a, #32; addl r, #32
965 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
966 // on 32bit in the case of cmpl, the imm might be an oop
967 tail_size = 4;
968 debug_only(has_disp32 = true); // has both kinds of operands!
969 break;
970
971 case 0x83: // addl a, #8; addl r, #8
972 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
973 debug_only(has_disp32 = true); // has both kinds of operands!
974 tail_size = 1;
975 break;
976
977 case 0x15: // adc rax, #32
978 case 0x05: // add rax, #32
979 case 0x25: // and rax, #32
980 case 0x3D: // cmp rax, #32
981 case 0x0D: // or rax, #32
982 case 0x1D: // sbb rax, #32
983 case 0x2D: // sub rax, #32
984 case 0x35: // xor rax, #32
985 return which == end_pc_operand ? ip + 4 : ip;
986
987 case 0x9B:
988 switch (0xFF & *ip++) {
989 case 0xD9: // fnstcw a
990 debug_only(has_disp32 = true);
991 break;
992 default:
993 ShouldNotReachHere();
994 }
995 break;
996
997 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
998 case REP4(0x10): // adc...
999 case REP4(0x20): // and...
1000 case REP4(0x30): // xor...
1001 case REP4(0x08): // or...
1002 case REP4(0x18): // sbb...
1003 case REP4(0x28): // sub...
1004 case 0xF7: // mull a
1005 case 0x8D: // lea r, a
1006 case 0x87: // xchg r, a
1007 case REP4(0x38): // cmp...
1008 case 0x85: // test r, a
1009 debug_only(has_disp32 = true); // has both kinds of operands!
1010 break;
1011
1012 case 0xA8: // testb rax, #8
1013 return which == end_pc_operand ? ip + 1 : ip;
1014 case 0xA9: // testl/testq rax, #32
1015 return which == end_pc_operand ? ip + 4 : ip;
1016
1017 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
1018 case 0xC6: // movb a, #8
1019 case 0x80: // cmpb a, #8
1020 case 0x6B: // imul r, a, #8
1021 debug_only(has_disp32 = true); // has both kinds of operands!
1022 tail_size = 1; // the imm8
1023 break;
1024
1025 case 0xC4: // VEX_3bytes
1026 case 0xC5: // VEX_2bytes
1027 assert((UseAVX > 0), "shouldn't have VEX prefix");
1028 assert(ip == inst+1, "no prefixes allowed");
1029 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
1030 // but they have prefix 0x0F and processed when 0x0F processed above.
1031 //
1032 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
1033 // instructions (these instructions are not supported in 64-bit mode).
1034 // To distinguish them bits [7:6] are set in the VEX second byte since
1035 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
1036 // those VEX bits REX and vvvv bits are inverted.
1037 //
1038 // Fortunately C2 doesn't generate these instructions so we don't need
1039 // to check for them in product version.
1040
1041 // Check second byte
1042 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
1043
1044 int vex_opcode;
1045 // First byte
1046 if ((0xFF & *inst) == VEX_3bytes) {
1047 vex_opcode = VEX_OPCODE_MASK & *ip;
1048 ip++; // third byte
1049 is_64bit = ((VEX_W & *ip) == VEX_W);
1050 } else {
1051 vex_opcode = VEX_OPCODE_0F;
1052 }
1053 ip++; // opcode
1054 // To find the end of instruction (which == end_pc_operand).
1055 switch (vex_opcode) {
1056 case VEX_OPCODE_0F:
1057 switch (0xFF & *ip) {
1058 case 0x70: // pshufd r, r/a, #8
1059 case 0x71: // ps[rl|ra|ll]w r, #8
1060 case 0x72: // ps[rl|ra|ll]d r, #8
1061 case 0x73: // ps[rl|ra|ll]q r, #8
1062 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
1063 case 0xC4: // pinsrw r, r, r/a, #8
1064 case 0xC5: // pextrw r/a, r, #8
1065 case 0xC6: // shufp[s|d] r, r, r/a, #8
1066 tail_size = 1; // the imm8
1067 break;
1068 }
1069 break;
1070 case VEX_OPCODE_0F_3A:
1071 tail_size = 1;
1072 break;
1073 }
1074 ip++; // skip opcode
1075 debug_only(has_disp32 = true); // has both kinds of operands!
1076 break;
1077
1078 case 0x62: // EVEX_4bytes
1079 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
1080 assert(ip == inst+1, "no prefixes allowed");
1081 // no EVEX collisions, all instructions that have 0x62 opcodes
1082 // have EVEX versions and are subopcodes of 0x66
1083 ip++; // skip P0 and examine W in P1
1084 is_64bit = ((VEX_W & *ip) == VEX_W);
1085 ip++; // move to P2
1086 ip++; // skip P2, move to opcode
1087 // To find the end of instruction (which == end_pc_operand).
1088 switch (0xFF & *ip) {
1089 case 0x22: // pinsrd r, r/a, #8
1090 case 0x61: // pcmpestri r, r/a, #8
1091 case 0x70: // pshufd r, r/a, #8
1092 case 0x73: // psrldq r, #8
1093 case 0x1f: // evpcmpd/evpcmpq
1094 case 0x3f: // evpcmpb/evpcmpw
1095 tail_size = 1; // the imm8
1096 break;
1097 default:
1098 break;
1099 }
1100 ip++; // skip opcode
1101 debug_only(has_disp32 = true); // has both kinds of operands!
1102 break;
1103
1104 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
1105 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
1106 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
1107 case 0xDD: // fld_d a; fst_d a; fstp_d a
1108 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
1109 case 0xDF: // fild_d a; fistp_d a
1110 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
1111 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
1112 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
1113 debug_only(has_disp32 = true);
1114 break;
1115
1116 case 0xE8: // call rdisp32
1117 case 0xE9: // jmp rdisp32
1118 if (which == end_pc_operand) return ip + 4;
1119 assert(which == call32_operand, "call has no disp32 or imm");
1120 return ip;
1121
1122 case 0xF0: // Lock
1123 goto again_after_prefix;
1124
1125 case 0xF3: // For SSE
1126 case 0xF2: // For SSE2
1127 switch (0xFF & *ip++) {
1128 case REX:
1129 case REX_B:
1130 case REX_X:
1131 case REX_XB:
1132 case REX_R:
1133 case REX_RB:
1134 case REX_RX:
1135 case REX_RXB:
1136 case REX_W:
1137 case REX_WB:
1138 case REX_WX:
1139 case REX_WXB:
1140 case REX_WR:
1141 case REX_WRB:
1142 case REX_WRX:
1143 case REX_WRXB:
1144 NOT_LP64(assert(false, "found 64bit prefix"));
1145 ip++;
1146 default:
1147 ip++;
1148 }
1149 debug_only(has_disp32 = true); // has both kinds of operands!
1150 break;
1151
1152 default:
1153 ShouldNotReachHere();
1154
1155 #undef REP8
1156 #undef REP16
1157 }
1158
1159 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1160 #ifdef _LP64
1161 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1162 #else
1163 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1164 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1165 #endif // LP64
1166 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1167
1168 // parse the output of emit_operand
1169 int op2 = 0xFF & *ip++;
1170 int base = op2 & 0x07;
1171 int op3 = -1;
1172 const int b100 = 4;
1173 const int b101 = 5;
1174 if (base == b100 && (op2 >> 6) != 3) {
1175 op3 = 0xFF & *ip++;
1176 base = op3 & 0x07; // refetch the base
1177 }
1178 // now ip points at the disp (if any)
1179
1180 switch (op2 >> 6) {
1181 case 0:
1182 // [00 reg 100][ss index base]
1183 // [00 reg 100][00 100 esp]
1184 // [00 reg base]
1185 // [00 reg 100][ss index 101][disp32]
1186 // [00 reg 101] [disp32]
1187
1188 if (base == b101) {
1189 if (which == disp32_operand)
1190 return ip; // caller wants the disp32
1191 ip += 4; // skip the disp32
1192 }
1193 break;
1194
1195 case 1:
1196 // [01 reg 100][ss index base][disp8]
1197 // [01 reg 100][00 100 esp][disp8]
1198 // [01 reg base] [disp8]
1199 ip += 1; // skip the disp8
1200 break;
1201
1202 case 2:
1203 // [10 reg 100][ss index base][disp32]
1204 // [10 reg 100][00 100 esp][disp32]
1205 // [10 reg base] [disp32]
1206 if (which == disp32_operand)
1207 return ip; // caller wants the disp32
1208 ip += 4; // skip the disp32
1209 break;
1210
1211 case 3:
1212 // [11 reg base] (not a memory addressing mode)
1213 break;
1214 }
1215
1216 if (which == end_pc_operand) {
1217 return ip + tail_size;
1218 }
1219
1220 #ifdef _LP64
1221 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1222 #else
1223 assert(which == imm_operand, "instruction has only an imm field");
1224 #endif // LP64
1225 return ip;
1226 }
1227
1228 address Assembler::locate_next_instruction(address inst) {
1229 // Secretly share code with locate_operand:
1230 return locate_operand(inst, end_pc_operand);
1231 }
1232
1233
1234 #ifdef ASSERT
1235 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1236 address inst = inst_mark();
1237 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1238 address opnd;
1239
1240 Relocation* r = rspec.reloc();
1241 if (r->type() == relocInfo::none) {
1242 return;
1243 } else if (r->is_call() || format == call32_operand) {
1244 // assert(format == imm32_operand, "cannot specify a nonzero format");
1245 opnd = locate_operand(inst, call32_operand);
1246 } else if (r->is_data()) {
1247 assert(format == imm_operand || format == disp32_operand
1248 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1249 opnd = locate_operand(inst, (WhichOperand)format);
1250 } else {
1251 assert(format == imm_operand, "cannot specify a format");
1252 return;
1253 }
1254 assert(opnd == pc(), "must put operand where relocs can find it");
1255 }
1256 #endif // ASSERT
1257
1258 void Assembler::emit_operand(Register reg, Address adr, int post_addr_length) {
1259 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1260 }
1261
1262 void Assembler::emit_operand(XMMRegister reg, Address adr, int post_addr_length) {
1263 if (adr.isxmmindex()) {
1264 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec, post_addr_length);
1265 } else {
1266 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
1267 }
1268 }
1269
1270 // Now the Assembler instructions (identical for 32/64 bits)
1271
1272 void Assembler::adcl(Address dst, int32_t imm32) {
1273 InstructionMark im(this);
1274 prefix(dst);
1275 emit_arith_operand(0x81, rdx, dst, imm32);
1276 }
1277
1278 void Assembler::adcl(Address dst, Register src) {
1279 InstructionMark im(this);
1280 prefix(dst, src);
1281 emit_int8(0x11);
1282 emit_operand(src, dst, 0);
1283 }
1284
1285 void Assembler::adcl(Register dst, int32_t imm32) {
1286 prefix(dst);
1287 emit_arith(0x81, 0xD0, dst, imm32);
1288 }
1289
1290 void Assembler::adcl(Register dst, Address src) {
1291 InstructionMark im(this);
1292 prefix(src, dst);
1293 emit_int8(0x13);
1294 emit_operand(dst, src, 0);
1295 }
1296
1297 void Assembler::adcl(Register dst, Register src) {
1298 (void) prefix_and_encode(dst->encoding(), src->encoding());
1299 emit_arith(0x13, 0xC0, dst, src);
1300 }
1301
1302 void Assembler::addl(Address dst, int32_t imm32) {
1303 InstructionMark im(this);
1304 prefix(dst);
1305 emit_arith_operand(0x81, rax, dst, imm32);
1306 }
1307
1308 void Assembler::addb(Address dst, int imm8) {
1309 InstructionMark im(this);
1310 prefix(dst);
1311 emit_int8((unsigned char)0x80);
1312 emit_operand(rax, dst, 1);
1313 emit_int8(imm8);
1314 }
1315
1316 void Assembler::addw(Register dst, Register src) {
1317 emit_int8(0x66);
1318 (void)prefix_and_encode(dst->encoding(), src->encoding());
1319 emit_arith(0x03, 0xC0, dst, src);
1320 }
1321
1322 void Assembler::addw(Address dst, int imm16) {
1323 InstructionMark im(this);
1324 emit_int8(0x66);
1325 prefix(dst);
1326 emit_int8((unsigned char)0x81);
1327 emit_operand(rax, dst, 2);
1328 emit_int16(imm16);
1329 }
1330
1331 void Assembler::addl(Address dst, Register src) {
1332 InstructionMark im(this);
1333 prefix(dst, src);
1334 emit_int8(0x01);
1335 emit_operand(src, dst, 0);
1336 }
1337
1338 void Assembler::addl(Register dst, int32_t imm32) {
1339 prefix(dst);
1340 emit_arith(0x81, 0xC0, dst, imm32);
1341 }
1342
1343 void Assembler::addl(Register dst, Address src) {
1344 InstructionMark im(this);
1345 prefix(src, dst);
1346 emit_int8(0x03);
1347 emit_operand(dst, src, 0);
1348 }
1349
1350 void Assembler::addl(Register dst, Register src) {
1351 (void) prefix_and_encode(dst->encoding(), src->encoding());
1352 emit_arith(0x03, 0xC0, dst, src);
1353 }
1354
1355 void Assembler::addr_nop_4() {
1356 assert(UseAddressNop, "no CPU support");
1357 // 4 bytes: NOP DWORD PTR [EAX+0]
1358 emit_int32(0x0F,
1359 0x1F,
1360 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1361 0); // 8-bits offset (1 byte)
1362 }
1363
1364 void Assembler::addr_nop_5() {
1365 assert(UseAddressNop, "no CPU support");
1366 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1367 emit_int32(0x0F,
1368 0x1F,
1369 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1370 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1371 emit_int8(0); // 8-bits offset (1 byte)
1372 }
1373
1374 void Assembler::addr_nop_7() {
1375 assert(UseAddressNop, "no CPU support");
1376 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1377 emit_int24(0x0F,
1378 0x1F,
1379 (unsigned char)0x80);
1380 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1381 emit_int32(0); // 32-bits offset (4 bytes)
1382 }
1383
1384 void Assembler::addr_nop_8() {
1385 assert(UseAddressNop, "no CPU support");
1386 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1387 emit_int32(0x0F,
1388 0x1F,
1389 (unsigned char)0x84,
1390 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1391 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1392 emit_int32(0); // 32-bits offset (4 bytes)
1393 }
1394
1395 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1396 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1397 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1398 attributes.set_rex_vex_w_reverted();
1399 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1400 emit_int16(0x58, (0xC0 | encode));
1401 }
1402
1403 void Assembler::addsd(XMMRegister dst, Address src) {
1404 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1405 InstructionMark im(this);
1406 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1407 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1408 attributes.set_rex_vex_w_reverted();
1409 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1410 emit_int8(0x58);
1411 emit_operand(dst, src, 0);
1412 }
1413
1414 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1415 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1416 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1417 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1418 emit_int16(0x58, (0xC0 | encode));
1419 }
1420
1421 void Assembler::addss(XMMRegister dst, Address src) {
1422 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1423 InstructionMark im(this);
1424 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1425 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1426 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1427 emit_int8(0x58);
1428 emit_operand(dst, src, 0);
1429 }
1430
1431 void Assembler::aesdec(XMMRegister dst, Address src) {
1432 assert(VM_Version::supports_aes(), "");
1433 InstructionMark im(this);
1434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1435 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1436 emit_int8((unsigned char)0xDE);
1437 emit_operand(dst, src, 0);
1438 }
1439
1440 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1441 assert(VM_Version::supports_aes(), "");
1442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1443 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1444 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1445 }
1446
1447 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1448 assert(VM_Version::supports_avx512_vaes(), "");
1449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1450 attributes.set_is_evex_instruction();
1451 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1452 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1453 }
1454
1455
1456 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1457 assert(VM_Version::supports_aes(), "");
1458 InstructionMark im(this);
1459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1460 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1461 emit_int8((unsigned char)0xDF);
1462 emit_operand(dst, src, 0);
1463 }
1464
1465 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1466 assert(VM_Version::supports_aes(), "");
1467 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1468 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1469 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1470 }
1471
1472 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1473 assert(VM_Version::supports_avx512_vaes(), "");
1474 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1475 attributes.set_is_evex_instruction();
1476 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1477 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1478 }
1479
1480 void Assembler::aesenc(XMMRegister dst, Address src) {
1481 assert(VM_Version::supports_aes(), "");
1482 InstructionMark im(this);
1483 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1484 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1485 emit_int8((unsigned char)0xDC);
1486 emit_operand(dst, src, 0);
1487 }
1488
1489 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1490 assert(VM_Version::supports_aes(), "");
1491 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1492 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1493 emit_int16((unsigned char)0xDC, 0xC0 | encode);
1494 }
1495
1496 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1497 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1498 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1499 attributes.set_is_evex_instruction();
1500 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1501 emit_int16((unsigned char)0xDC, (0xC0 | encode));
1502 }
1503
1504 void Assembler::aesenclast(XMMRegister dst, Address src) {
1505 assert(VM_Version::supports_aes(), "");
1506 InstructionMark im(this);
1507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1508 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1509 emit_int8((unsigned char)0xDD);
1510 emit_operand(dst, src, 0);
1511 }
1512
1513 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1514 assert(VM_Version::supports_aes(), "");
1515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1517 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1518 }
1519
1520 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1521 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1522 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1523 attributes.set_is_evex_instruction();
1524 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1525 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1526 }
1527
1528 void Assembler::andb(Address dst, Register src) {
1529 InstructionMark im(this);
1530 prefix(dst, src, true);
1531 emit_int8(0x20);
1532 emit_operand(src, dst, 0);
1533 }
1534
1535 void Assembler::andw(Register dst, Register src) {
1536 (void)prefix_and_encode(dst->encoding(), src->encoding());
1537 emit_arith(0x23, 0xC0, dst, src);
1538 }
1539
1540 void Assembler::andl(Address dst, int32_t imm32) {
1541 InstructionMark im(this);
1542 prefix(dst);
1543 emit_arith_operand(0x81, as_Register(4), dst, imm32);
1544 }
1545
1546 void Assembler::andl(Register dst, int32_t imm32) {
1547 prefix(dst);
1548 emit_arith(0x81, 0xE0, dst, imm32);
1549 }
1550
1551 void Assembler::andl(Address dst, Register src) {
1552 InstructionMark im(this);
1553 prefix(dst, src);
1554 emit_int8(0x21);
1555 emit_operand(src, dst, 0);
1556 }
1557
1558 void Assembler::andl(Register dst, Address src) {
1559 InstructionMark im(this);
1560 prefix(src, dst);
1561 emit_int8(0x23);
1562 emit_operand(dst, src, 0);
1563 }
1564
1565 void Assembler::andl(Register dst, Register src) {
1566 (void) prefix_and_encode(dst->encoding(), src->encoding());
1567 emit_arith(0x23, 0xC0, dst, src);
1568 }
1569
1570 void Assembler::andnl(Register dst, Register src1, Register src2) {
1571 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1572 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1573 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1574 emit_int16((unsigned char)0xF2, (0xC0 | encode));
1575 }
1576
1577 void Assembler::andnl(Register dst, Register src1, Address src2) {
1578 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1579 InstructionMark im(this);
1580 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1581 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1582 emit_int8((unsigned char)0xF2);
1583 emit_operand(dst, src2, 0);
1584 }
1585
1586 void Assembler::bsfl(Register dst, Register src) {
1587 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1588 emit_int24(0x0F,
1589 (unsigned char)0xBC,
1590 0xC0 | encode);
1591 }
1592
1593 void Assembler::bsrl(Register dst, Register src) {
1594 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1595 emit_int24(0x0F,
1596 (unsigned char)0xBD,
1597 0xC0 | encode);
1598 }
1599
1600 void Assembler::bswapl(Register reg) { // bswap
1601 int encode = prefix_and_encode(reg->encoding());
1602 emit_int16(0x0F, (0xC8 | encode));
1603 }
1604
1605 void Assembler::blsil(Register dst, Register src) {
1606 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1607 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1608 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1609 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1610 }
1611
1612 void Assembler::blsil(Register dst, Address src) {
1613 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1614 InstructionMark im(this);
1615 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1616 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1617 emit_int8((unsigned char)0xF3);
1618 emit_operand(rbx, src, 0);
1619 }
1620
1621 void Assembler::blsmskl(Register dst, Register src) {
1622 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1623 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1624 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1625 emit_int16((unsigned char)0xF3,
1626 0xC0 | encode);
1627 }
1628
1629 void Assembler::blsmskl(Register dst, Address src) {
1630 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1631 InstructionMark im(this);
1632 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1633 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1634 emit_int8((unsigned char)0xF3);
1635 emit_operand(rdx, src, 0);
1636 }
1637
1638 void Assembler::blsrl(Register dst, Register src) {
1639 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1640 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1641 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1642 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1643 }
1644
1645 void Assembler::blsrl(Register dst, Address src) {
1646 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1647 InstructionMark im(this);
1648 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1649 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1650 emit_int8((unsigned char)0xF3);
1651 emit_operand(rcx, src, 0);
1652 }
1653
1654 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1655 // suspect disp32 is always good
1656 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1657
1658 if (L.is_bound()) {
1659 const int long_size = 5;
1660 int offs = (int)( target(L) - pc() );
1661 assert(offs <= 0, "assembler error");
1662 InstructionMark im(this);
1663 // 1110 1000 #32-bit disp
1664 emit_int8((unsigned char)0xE8);
1665 emit_data(offs - long_size, rtype, operand);
1666 } else {
1667 InstructionMark im(this);
1668 // 1110 1000 #32-bit disp
1669 L.add_patch_at(code(), locator());
1670
1671 emit_int8((unsigned char)0xE8);
1672 emit_data(int(0), rtype, operand);
1673 }
1674 }
1675
1676 void Assembler::call(Register dst) {
1677 int encode = prefix_and_encode(dst->encoding());
1678 emit_int16((unsigned char)0xFF, (0xD0 | encode));
1679 }
1680
1681
1682 void Assembler::call(Address adr) {
1683 InstructionMark im(this);
1684 prefix(adr);
1685 emit_int8((unsigned char)0xFF);
1686 emit_operand(rdx, adr, 0);
1687 }
1688
1689 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1690 InstructionMark im(this);
1691 emit_int8((unsigned char)0xE8);
1692 intptr_t disp = entry - (pc() + sizeof(int32_t));
1693 // Entry is NULL in case of a scratch emit.
1694 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1695 // Technically, should use call32_operand, but this format is
1696 // implied by the fact that we're emitting a call instruction.
1697
1698 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1699 emit_data((int) disp, rspec, operand);
1700 }
1701
1702 void Assembler::cdql() {
1703 emit_int8((unsigned char)0x99);
1704 }
1705
1706 void Assembler::cld() {
1707 emit_int8((unsigned char)0xFC);
1708 }
1709
1710 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1711 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1712 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1713 emit_int24(0x0F,
1714 0x40 | cc,
1715 0xC0 | encode);
1716 }
1717
1718
1719 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1720 InstructionMark im(this);
1721 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1722 prefix(src, dst);
1723 emit_int16(0x0F, (0x40 | cc));
1724 emit_operand(dst, src, 0);
1725 }
1726
1727 void Assembler::cmpb(Address dst, int imm8) {
1728 InstructionMark im(this);
1729 prefix(dst);
1730 emit_int8((unsigned char)0x80);
1731 emit_operand(rdi, dst, 1);
1732 emit_int8(imm8);
1733 }
1734
1735 void Assembler::cmpl(Address dst, int32_t imm32) {
1736 InstructionMark im(this);
1737 prefix(dst);
1738 emit_arith_operand(0x81, as_Register(7), dst, imm32);
1739 }
1740
1741 void Assembler::cmpl(Register dst, int32_t imm32) {
1742 prefix(dst);
1743 emit_arith(0x81, 0xF8, dst, imm32);
1744 }
1745
1746 void Assembler::cmpl(Register dst, Register src) {
1747 (void) prefix_and_encode(dst->encoding(), src->encoding());
1748 emit_arith(0x3B, 0xC0, dst, src);
1749 }
1750
1751 void Assembler::cmpl(Register dst, Address src) {
1752 InstructionMark im(this);
1753 prefix(src, dst);
1754 emit_int8(0x3B);
1755 emit_operand(dst, src, 0);
1756 }
1757
1758 void Assembler::cmpl_imm32(Address dst, int32_t imm32) {
1759 InstructionMark im(this);
1760 prefix(dst);
1761 emit_arith_operand_imm32(0x81, as_Register(7), dst, imm32);
1762 }
1763
1764 void Assembler::cmpw(Address dst, int imm16) {
1765 InstructionMark im(this);
1766 emit_int8(0x66);
1767 prefix(dst);
1768 emit_int8((unsigned char)0x81);
1769 emit_operand(rdi, dst, 2);
1770 emit_int16(imm16);
1771 }
1772
1773 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1774 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1775 // The ZF is set if the compared values were equal, and cleared otherwise.
1776 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1777 InstructionMark im(this);
1778 prefix(adr, reg);
1779 emit_int16(0x0F, (unsigned char)0xB1);
1780 emit_operand(reg, adr, 0);
1781 }
1782
1783 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
1784 InstructionMark im(this);
1785 size_prefix();
1786 prefix(adr, reg);
1787 emit_int16(0x0F, (unsigned char)0xB1);
1788 emit_operand(reg, adr, 0);
1789 }
1790
1791 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1792 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1793 // The ZF is set if the compared values were equal, and cleared otherwise.
1794 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1795 InstructionMark im(this);
1796 prefix(adr, reg, true);
1797 emit_int16(0x0F, (unsigned char)0xB0);
1798 emit_operand(reg, adr, 0);
1799 }
1800
1801 void Assembler::comisd(XMMRegister dst, Address src) {
1802 // NOTE: dbx seems to decode this as comiss even though the
1803 // 0x66 is there. Strangely ucomisd comes out correct
1804 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1805 InstructionMark im(this);
1806 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1807 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1808 attributes.set_rex_vex_w_reverted();
1809 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1810 emit_int8(0x2F);
1811 emit_operand(dst, src, 0);
1812 }
1813
1814 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1815 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1816 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1817 attributes.set_rex_vex_w_reverted();
1818 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1819 emit_int16(0x2F, (0xC0 | encode));
1820 }
1821
1822 void Assembler::comiss(XMMRegister dst, Address src) {
1823 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1824 InstructionMark im(this);
1825 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1826 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1827 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1828 emit_int8(0x2F);
1829 emit_operand(dst, src, 0);
1830 }
1831
1832 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1833 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1834 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1835 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1836 emit_int16(0x2F, (0xC0 | encode));
1837 }
1838
1839 void Assembler::cpuid() {
1840 emit_int16(0x0F, (unsigned char)0xA2);
1841 }
1842
1843 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1844 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1845 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1846 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1847 //
1848 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1849 //
1850 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1851 //
1852 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1853 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1854 assert(VM_Version::supports_sse4_2(), "");
1855 int8_t w = 0x01;
1856 Prefix p = Prefix_EMPTY;
1857
1858 emit_int8((unsigned char)0xF2);
1859 switch (sizeInBytes) {
1860 case 1:
1861 w = 0;
1862 break;
1863 case 2:
1864 case 4:
1865 break;
1866 LP64_ONLY(case 8:)
1867 // This instruction is not valid in 32 bits
1868 // Note:
1869 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1870 //
1871 // Page B - 72 Vol. 2C says
1872 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1873 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1874 // F0!!!
1875 // while 3 - 208 Vol. 2A
1876 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1877 //
1878 // the 0 on a last bit is reserved for a different flavor of this instruction :
1879 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1880 p = REX_W;
1881 break;
1882 default:
1883 assert(0, "Unsupported value for a sizeInBytes argument");
1884 break;
1885 }
1886 LP64_ONLY(prefix(crc, v, p);)
1887 emit_int32(0x0F,
1888 0x38,
1889 0xF0 | w,
1890 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1891 }
1892
1893 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1894 assert(VM_Version::supports_sse4_2(), "");
1895 InstructionMark im(this);
1896 int8_t w = 0x01;
1897 Prefix p = Prefix_EMPTY;
1898
1899 emit_int8((int8_t)0xF2);
1900 switch (sizeInBytes) {
1901 case 1:
1902 w = 0;
1903 break;
1904 case 2:
1905 case 4:
1906 break;
1907 LP64_ONLY(case 8:)
1908 // This instruction is not valid in 32 bits
1909 p = REX_W;
1910 break;
1911 default:
1912 assert(0, "Unsupported value for a sizeInBytes argument");
1913 break;
1914 }
1915 LP64_ONLY(prefix(crc, adr, p);)
1916 emit_int24(0x0F, 0x38, (0xF0 | w));
1917 emit_operand(crc, adr, 0);
1918 }
1919
1920 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1921 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1922 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1923 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1924 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1925 }
1926
1927 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
1928 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1929 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1930 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1931 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1932 }
1933
1934 void Assembler::vcvtps2ph(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
1935 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
1936 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
1937 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
1938 emit_int24(0x1D, (0xC0 | encode), imm8);
1939 }
1940
1941 void Assembler::evcvtps2ph(Address dst, KRegister mask, XMMRegister src, int imm8, int vector_len) {
1942 assert(VM_Version::supports_evex(), "");
1943 InstructionMark im(this);
1944 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
1945 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_64bit);
1946 attributes.reset_is_clear_context();
1947 attributes.set_embedded_opmask_register_specifier(mask);
1948 attributes.set_is_evex_instruction();
1949 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
1950 emit_int8(0x1D);
1951 emit_operand(src, dst, 1);
1952 emit_int8(imm8);
1953 }
1954
1955 void Assembler::vcvtps2ph(Address dst, XMMRegister src, int imm8, int vector_len) {
1956 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
1957 InstructionMark im(this);
1958 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /*uses_vl */ true);
1959 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
1960 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
1961 emit_int8(0x1D);
1962 emit_operand(src, dst, 1);
1963 emit_int8(imm8);
1964 }
1965
1966 void Assembler::vcvtph2ps(XMMRegister dst, XMMRegister src, int vector_len) {
1967 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
1968 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ true);
1969 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1970 emit_int16(0x13, (0xC0 | encode));
1971 }
1972
1973 void Assembler::vcvtph2ps(XMMRegister dst, Address src, int vector_len) {
1974 assert(VM_Version::supports_evex() || VM_Version::supports_f16c(), "");
1975 InstructionMark im(this);
1976 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /*uses_vl */ true);
1977 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
1978 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1979 emit_int8(0x13);
1980 emit_operand(dst, src, 0);
1981 }
1982
1983 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1984 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1985 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1986 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1987 emit_int16(0x5B, (0xC0 | encode));
1988 }
1989
1990 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
1991 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1992 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1993 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1994 emit_int16(0x5B, (0xC0 | encode));
1995 }
1996
1997 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1998 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1999 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2000 attributes.set_rex_vex_w_reverted();
2001 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2002 emit_int16(0x5A, (0xC0 | encode));
2003 }
2004
2005 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
2006 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2007 InstructionMark im(this);
2008 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2009 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2010 attributes.set_rex_vex_w_reverted();
2011 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2012 emit_int8(0x5A);
2013 emit_operand(dst, src, 0);
2014 }
2015
2016 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
2017 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2018 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2019 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2020 emit_int16(0x2A, (0xC0 | encode));
2021 }
2022
2023 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
2024 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2025 InstructionMark im(this);
2026 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2027 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2028 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2029 emit_int8(0x2A);
2030 emit_operand(dst, src, 0);
2031 }
2032
2033 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
2034 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2035 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2036 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2037 emit_int16(0x2A, (0xC0 | encode));
2038 }
2039
2040 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
2041 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2042 InstructionMark im(this);
2043 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2044 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2045 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2046 emit_int8(0x2A);
2047 emit_operand(dst, src, 0);
2048 }
2049
2050 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
2051 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2052 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2053 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2054 emit_int16(0x2A, (0xC0 | encode));
2055 }
2056
2057 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
2058 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2059 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2060 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2061 emit_int16(0x5A, (0xC0 | encode));
2062 }
2063
2064 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
2065 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2066 InstructionMark im(this);
2067 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2068 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2069 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2070 emit_int8(0x5A);
2071 emit_operand(dst, src, 0);
2072 }
2073
2074
2075 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
2076 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2077 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2078 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2079 emit_int16(0x2C, (0xC0 | encode));
2080 }
2081
2082 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
2083 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2084 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2085 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2086 emit_int16(0x2D, (0xC0 | encode));
2087 }
2088
2089 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
2090 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2091 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2092 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2093 emit_int16(0x2C, (0xC0 | encode));
2094 }
2095
2096 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
2097 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2098 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2099 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2100 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2101 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2102 }
2103
2104 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
2105 assert(VM_Version::supports_ssse3(), "");
2106 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2107 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2108 emit_int16(0x1C, (0xC0 | encode));
2109 }
2110
2111 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
2112 assert(VM_Version::supports_ssse3(), "");
2113 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2114 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2115 emit_int16(0x1D, (0xC0 | encode));
2116 }
2117
2118 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
2119 assert(VM_Version::supports_ssse3(), "");
2120 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2121 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2122 emit_int16(0x1E, (0xC0 | encode));
2123 }
2124
2125 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
2126 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2127 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2128 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
2129 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2130 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2131 emit_int16(0x1C, (0xC0 | encode));
2132 }
2133
2134 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
2135 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2136 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2137 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
2138 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2139 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2140 emit_int16(0x1D, (0xC0 | encode));
2141 }
2142
2143 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
2144 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
2145 vector_len == AVX_256bit? VM_Version::supports_avx2() :
2146 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
2147 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2148 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2149 emit_int16(0x1E, (0xC0 | encode));
2150 }
2151
2152 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
2153 assert(UseAVX > 2, "");
2154 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2155 attributes.set_is_evex_instruction();
2156 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2157 emit_int16(0x1F, (0xC0 | encode));
2158 }
2159
2160 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2161 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2162 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2164 emit_int16(0x5A, (0xC0 | encode));
2165 }
2166
2167 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2168 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2169 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2170 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2171 attributes.set_rex_vex_w_reverted();
2172 emit_int16(0x5A, (0xC0 | encode));
2173 }
2174
2175 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2176 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2177 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2178 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2179 emit_int16(0x5B, (0xC0 | encode));
2180 }
2181
2182 void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2183 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2184 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2185 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2186 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2187 }
2188
2189 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2190 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2191 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2192 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2193 emit_int16(0x5B, (0xC0 | encode));
2194 }
2195
2196 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2197 assert(VM_Version::supports_avx512dq(), "");
2198 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2199 attributes.set_is_evex_instruction();
2200 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2201 emit_int16(0x7A, (0xC0 | encode));
2202 }
2203
2204 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2205 assert(VM_Version::supports_avx512dq(), "");
2206 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2207 attributes.set_is_evex_instruction();
2208 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2209 emit_int16(0x7B, (0xC0 | encode));
2210 }
2211
2212 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2213 assert(VM_Version::supports_avx512dq(), "");
2214 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2215 attributes.set_is_evex_instruction();
2216 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2217 emit_int16(0x5B, (0xC0 | encode));
2218 }
2219
2220 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2221 assert(VM_Version::supports_avx512dq(), "");
2222 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2223 attributes.set_is_evex_instruction();
2224 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2225 emit_int16(0x7A, (0xC0 | encode));
2226 }
2227
2228 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2229 assert(VM_Version::supports_avx512dq(), "");
2230 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2231 attributes.set_is_evex_instruction();
2232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2233 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2234 }
2235
2236 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
2237 assert(VM_Version::supports_avx512bw(), "");
2238 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2239 attributes.set_is_evex_instruction();
2240 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2241 emit_int16(0x30, (0xC0 | encode));
2242 }
2243
2244 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
2245 assert(UseAVX > 2, "");
2246 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2247 attributes.set_is_evex_instruction();
2248 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2249 emit_int16(0x33, (0xC0 | encode));
2250 }
2251
2252 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
2253 assert(UseAVX > 2, "");
2254 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2255 attributes.set_is_evex_instruction();
2256 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2257 emit_int16(0x31, (0xC0 | encode));
2258 }
2259
2260 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
2261 assert(UseAVX > 2, "");
2262 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2263 attributes.set_is_evex_instruction();
2264 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2265 emit_int16(0x35, (0xC0 | encode));
2266 }
2267
2268 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
2269 assert(UseAVX > 2, "");
2270 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2271 attributes.set_is_evex_instruction();
2272 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2273 emit_int16(0x32, (0xC0 | encode));
2274 }
2275
2276 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
2277 assert(UseAVX > 2, "");
2278 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2279 attributes.set_is_evex_instruction();
2280 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2281 emit_int16(0x34, (0xC0 | encode));
2282 }
2283
2284 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
2285 assert(UseAVX > 2, "");
2286 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2287 attributes.set_is_evex_instruction();
2288 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2289 emit_int16(0x25, (0xC0 | encode));
2290 }
2291
2292 void Assembler::decl(Address dst) {
2293 // Don't use it directly. Use MacroAssembler::decrement() instead.
2294 InstructionMark im(this);
2295 prefix(dst);
2296 emit_int8((unsigned char)0xFF);
2297 emit_operand(rcx, dst, 0);
2298 }
2299
2300 void Assembler::divsd(XMMRegister dst, Address src) {
2301 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2302 InstructionMark im(this);
2303 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2304 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2305 attributes.set_rex_vex_w_reverted();
2306 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2307 emit_int8(0x5E);
2308 emit_operand(dst, src, 0);
2309 }
2310
2311 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
2312 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2313 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2314 attributes.set_rex_vex_w_reverted();
2315 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2316 emit_int16(0x5E, (0xC0 | encode));
2317 }
2318
2319 void Assembler::divss(XMMRegister dst, Address src) {
2320 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2321 InstructionMark im(this);
2322 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2323 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2324 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2325 emit_int8(0x5E);
2326 emit_operand(dst, src, 0);
2327 }
2328
2329 void Assembler::divss(XMMRegister dst, XMMRegister src) {
2330 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2331 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2332 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2333 emit_int16(0x5E, (0xC0 | encode));
2334 }
2335
2336 void Assembler::hlt() {
2337 emit_int8((unsigned char)0xF4);
2338 }
2339
2340 void Assembler::idivl(Register src) {
2341 int encode = prefix_and_encode(src->encoding());
2342 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2343 }
2344
2345 void Assembler::divl(Register src) { // Unsigned
2346 int encode = prefix_and_encode(src->encoding());
2347 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2348 }
2349
2350 void Assembler::imull(Register src) {
2351 int encode = prefix_and_encode(src->encoding());
2352 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2353 }
2354
2355 void Assembler::imull(Register dst, Register src) {
2356 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2357 emit_int24(0x0F,
2358 (unsigned char)0xAF,
2359 (0xC0 | encode));
2360 }
2361
2362 void Assembler::imull(Register dst, Address src, int32_t value) {
2363 InstructionMark im(this);
2364 prefix(src, dst);
2365 if (is8bit(value)) {
2366 emit_int8((unsigned char)0x6B);
2367 emit_operand(dst, src, 1);
2368 emit_int8(value);
2369 } else {
2370 emit_int8((unsigned char)0x69);
2371 emit_operand(dst, src, 4);
2372 emit_int32(value);
2373 }
2374 }
2375
2376 void Assembler::imull(Register dst, Register src, int value) {
2377 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2378 if (is8bit(value)) {
2379 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2380 } else {
2381 emit_int16(0x69, (0xC0 | encode));
2382 emit_int32(value);
2383 }
2384 }
2385
2386 void Assembler::imull(Register dst, Address src) {
2387 InstructionMark im(this);
2388 prefix(src, dst);
2389 emit_int16(0x0F, (unsigned char)0xAF);
2390 emit_operand(dst, src, 0);
2391 }
2392
2393
2394 void Assembler::incl(Address dst) {
2395 // Don't use it directly. Use MacroAssembler::increment() instead.
2396 InstructionMark im(this);
2397 prefix(dst);
2398 emit_int8((unsigned char)0xFF);
2399 emit_operand(rax, dst, 0);
2400 }
2401
2402 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2403 InstructionMark im(this);
2404 assert((0 <= cc) && (cc < 16), "illegal cc");
2405 if (L.is_bound()) {
2406 address dst = target(L);
2407 assert(dst != NULL, "jcc most probably wrong");
2408
2409 const int short_size = 2;
2410 const int long_size = 6;
2411 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2412 if (maybe_short && is8bit(offs - short_size)) {
2413 // 0111 tttn #8-bit disp
2414 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2415 } else {
2416 // 0000 1111 1000 tttn #32-bit disp
2417 assert(is_simm32(offs - long_size),
2418 "must be 32bit offset (call4)");
2419 emit_int16(0x0F, (0x80 | cc));
2420 emit_int32(offs - long_size);
2421 }
2422 } else {
2423 // Note: could eliminate cond. jumps to this jump if condition
2424 // is the same however, seems to be rather unlikely case.
2425 // Note: use jccb() if label to be bound is very close to get
2426 // an 8-bit displacement
2427 L.add_patch_at(code(), locator());
2428 emit_int16(0x0F, (0x80 | cc));
2429 emit_int32(0);
2430 }
2431 }
2432
2433 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2434 if (L.is_bound()) {
2435 const int short_size = 2;
2436 address entry = target(L);
2437 #ifdef ASSERT
2438 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2439 intptr_t delta = short_branch_delta();
2440 if (delta != 0) {
2441 dist += (dist < 0 ? (-delta) :delta);
2442 }
2443 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2444 #endif
2445 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2446 // 0111 tttn #8-bit disp
2447 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2448 } else {
2449 InstructionMark im(this);
2450 L.add_patch_at(code(), locator(), file, line);
2451 emit_int16(0x70 | cc, 0);
2452 }
2453 }
2454
2455 void Assembler::jmp(Address adr) {
2456 InstructionMark im(this);
2457 prefix(adr);
2458 emit_int8((unsigned char)0xFF);
2459 emit_operand(rsp, adr, 0);
2460 }
2461
2462 void Assembler::jmp(Label& L, bool maybe_short) {
2463 if (L.is_bound()) {
2464 address entry = target(L);
2465 assert(entry != NULL, "jmp most probably wrong");
2466 InstructionMark im(this);
2467 const int short_size = 2;
2468 const int long_size = 5;
2469 intptr_t offs = entry - pc();
2470 if (maybe_short && is8bit(offs - short_size)) {
2471 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
2472 } else {
2473 emit_int8((unsigned char)0xE9);
2474 emit_int32(offs - long_size);
2475 }
2476 } else {
2477 // By default, forward jumps are always 32-bit displacements, since
2478 // we can't yet know where the label will be bound. If you're sure that
2479 // the forward jump will not run beyond 256 bytes, use jmpb to
2480 // force an 8-bit displacement.
2481 InstructionMark im(this);
2482 L.add_patch_at(code(), locator());
2483 emit_int8((unsigned char)0xE9);
2484 emit_int32(0);
2485 }
2486 }
2487
2488 void Assembler::jmp(Register entry) {
2489 int encode = prefix_and_encode(entry->encoding());
2490 emit_int16((unsigned char)0xFF, (0xE0 | encode));
2491 }
2492
2493 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2494 InstructionMark im(this);
2495 emit_int8((unsigned char)0xE9);
2496 assert(dest != NULL, "must have a target");
2497 intptr_t disp = dest - (pc() + sizeof(int32_t));
2498 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2499 emit_data(disp, rspec, call32_operand);
2500 }
2501
2502 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2503 if (L.is_bound()) {
2504 const int short_size = 2;
2505 address entry = target(L);
2506 assert(entry != NULL, "jmp most probably wrong");
2507 #ifdef ASSERT
2508 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2509 intptr_t delta = short_branch_delta();
2510 if (delta != 0) {
2511 dist += (dist < 0 ? (-delta) :delta);
2512 }
2513 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2514 #endif
2515 intptr_t offs = entry - pc();
2516 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
2517 } else {
2518 InstructionMark im(this);
2519 L.add_patch_at(code(), locator(), file, line);
2520 emit_int16((unsigned char)0xEB, 0);
2521 }
2522 }
2523
2524 void Assembler::ldmxcsr( Address src) {
2525 if (UseAVX > 0 ) {
2526 InstructionMark im(this);
2527 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2528 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2529 emit_int8((unsigned char)0xAE);
2530 emit_operand(as_Register(2), src, 0);
2531 } else {
2532 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2533 InstructionMark im(this);
2534 prefix(src);
2535 emit_int16(0x0F, (unsigned char)0xAE);
2536 emit_operand(as_Register(2), src, 0);
2537 }
2538 }
2539
2540 void Assembler::leal(Register dst, Address src) {
2541 InstructionMark im(this);
2542 prefix(src, dst);
2543 emit_int8((unsigned char)0x8D);
2544 emit_operand(dst, src, 0);
2545 }
2546
2547 void Assembler::lfence() {
2548 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
2549 }
2550
2551 void Assembler::lock() {
2552 emit_int8((unsigned char)0xF0);
2553 }
2554
2555 void Assembler::size_prefix() {
2556 emit_int8(0x66);
2557 }
2558
2559 void Assembler::lzcntl(Register dst, Register src) {
2560 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2561 emit_int8((unsigned char)0xF3);
2562 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2563 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
2564 }
2565
2566 void Assembler::lzcntl(Register dst, Address src) {
2567 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2568 InstructionMark im(this);
2569 emit_int8((unsigned char)0xF3);
2570 prefix(src, dst);
2571 emit_int16(0x0F, (unsigned char)0xBD);
2572 emit_operand(dst, src, 0);
2573 }
2574
2575 // Emit mfence instruction
2576 void Assembler::mfence() {
2577 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2578 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
2579 }
2580
2581 // Emit sfence instruction
2582 void Assembler::sfence() {
2583 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2584 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
2585 }
2586
2587 void Assembler::mov(Register dst, Register src) {
2588 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2589 }
2590
2591 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2592 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2593 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2594 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2595 attributes.set_rex_vex_w_reverted();
2596 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2597 emit_int16(0x28, (0xC0 | encode));
2598 }
2599
2600 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2601 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2602 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2603 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2604 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2605 emit_int16(0x28, (0xC0 | encode));
2606 }
2607
2608 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2609 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2611 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2612 emit_int16(0x16, (0xC0 | encode));
2613 }
2614
2615 void Assembler::movb(Register dst, Address src) {
2616 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2617 InstructionMark im(this);
2618 prefix(src, dst, true);
2619 emit_int8((unsigned char)0x8A);
2620 emit_operand(dst, src, 0);
2621 }
2622
2623 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2624 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2625 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2626 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2627 attributes.set_rex_vex_w_reverted();
2628 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2629 emit_int16(0x12, 0xC0 | encode);
2630 }
2631
2632 void Assembler::movddup(XMMRegister dst, Address src) {
2633 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2634 InstructionMark im(this);
2635 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2636 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
2637 attributes.set_rex_vex_w_reverted();
2638 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2639 emit_int8(0x12);
2640 emit_operand(dst, src, 0);
2641 }
2642
2643 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
2644 assert(VM_Version::supports_avx(), "");
2645 InstructionMark im(this);
2646 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2647 attributes.set_address_attributes(/* tuple_type */ EVEX_DUP, /* input_size_in_bits */ EVEX_64bit);
2648 attributes.set_rex_vex_w_reverted();
2649 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2650 emit_int8(0x12);
2651 emit_operand(dst, src, 0);
2652 }
2653
2654 void Assembler::kmovbl(KRegister dst, KRegister src) {
2655 assert(VM_Version::supports_avx512dq(), "");
2656 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2657 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2658 emit_int16((unsigned char)0x90, (0xC0 | encode));
2659 }
2660
2661 void Assembler::kmovbl(KRegister dst, Register src) {
2662 assert(VM_Version::supports_avx512dq(), "");
2663 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2664 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2665 emit_int16((unsigned char)0x92, (0xC0 | encode));
2666 }
2667
2668 void Assembler::kmovbl(Register dst, KRegister src) {
2669 assert(VM_Version::supports_avx512dq(), "");
2670 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2671 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2672 emit_int16((unsigned char)0x93, (0xC0 | encode));
2673 }
2674
2675 void Assembler::kmovwl(KRegister dst, Register src) {
2676 assert(VM_Version::supports_evex(), "");
2677 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2678 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2679 emit_int16((unsigned char)0x92, (0xC0 | encode));
2680 }
2681
2682 void Assembler::kmovwl(Register dst, KRegister src) {
2683 assert(VM_Version::supports_evex(), "");
2684 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2685 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2686 emit_int16((unsigned char)0x93, (0xC0 | encode));
2687 }
2688
2689 void Assembler::kmovwl(KRegister dst, Address src) {
2690 assert(VM_Version::supports_evex(), "");
2691 InstructionMark im(this);
2692 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2693 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2694 emit_int8((unsigned char)0x90);
2695 emit_operand(dst, src, 0);
2696 }
2697
2698 void Assembler::kmovwl(Address dst, KRegister src) {
2699 assert(VM_Version::supports_evex(), "");
2700 InstructionMark im(this);
2701 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2702 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2703 emit_int8((unsigned char)0x91);
2704 emit_operand(src, dst, 0);
2705 }
2706
2707 void Assembler::kmovwl(KRegister dst, KRegister src) {
2708 assert(VM_Version::supports_evex(), "");
2709 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2710 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2711 emit_int16((unsigned char)0x90, (0xC0 | encode));
2712 }
2713
2714 void Assembler::kmovdl(KRegister dst, Register src) {
2715 assert(VM_Version::supports_avx512bw(), "");
2716 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2717 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2718 emit_int16((unsigned char)0x92, (0xC0 | encode));
2719 }
2720
2721 void Assembler::kmovdl(Register dst, KRegister src) {
2722 assert(VM_Version::supports_avx512bw(), "");
2723 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2724 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2725 emit_int16((unsigned char)0x93, (0xC0 | encode));
2726 }
2727
2728 void Assembler::kmovql(KRegister dst, KRegister src) {
2729 assert(VM_Version::supports_avx512bw(), "");
2730 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2731 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2732 emit_int16((unsigned char)0x90, (0xC0 | encode));
2733 }
2734
2735 void Assembler::kmovql(KRegister dst, Address src) {
2736 assert(VM_Version::supports_avx512bw(), "");
2737 InstructionMark im(this);
2738 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2739 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2740 emit_int8((unsigned char)0x90);
2741 emit_operand(dst, src, 0);
2742 }
2743
2744 void Assembler::kmovql(Address dst, KRegister src) {
2745 assert(VM_Version::supports_avx512bw(), "");
2746 InstructionMark im(this);
2747 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2748 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2749 emit_int8((unsigned char)0x91);
2750 emit_operand(src, dst, 0);
2751 }
2752
2753 void Assembler::kmovql(KRegister dst, Register src) {
2754 assert(VM_Version::supports_avx512bw(), "");
2755 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2756 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2757 emit_int16((unsigned char)0x92, (0xC0 | encode));
2758 }
2759
2760 void Assembler::kmovql(Register dst, KRegister src) {
2761 assert(VM_Version::supports_avx512bw(), "");
2762 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2763 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2764 emit_int16((unsigned char)0x93, (0xC0 | encode));
2765 }
2766
2767 void Assembler::knotwl(KRegister dst, KRegister src) {
2768 assert(VM_Version::supports_evex(), "");
2769 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2770 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2771 emit_int16(0x44, (0xC0 | encode));
2772 }
2773
2774 void Assembler::knotbl(KRegister dst, KRegister src) {
2775 assert(VM_Version::supports_evex(), "");
2776 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2777 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2778 emit_int16(0x44, (0xC0 | encode));
2779 }
2780
2781 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
2782 assert(VM_Version::supports_avx512dq(), "");
2783 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2784 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2785 emit_int16(0x45, (0xC0 | encode));
2786 }
2787
2788 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
2789 assert(VM_Version::supports_evex(), "");
2790 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2791 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2792 emit_int16(0x45, (0xC0 | encode));
2793 }
2794
2795 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
2796 assert(VM_Version::supports_avx512bw(), "");
2797 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2798 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2799 emit_int16(0x45, (0xC0 | encode));
2800 }
2801
2802 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
2803 assert(VM_Version::supports_avx512bw(), "");
2804 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2805 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2806 emit_int16(0x45, (0xC0 | encode));
2807 }
2808
2809 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
2810 assert(VM_Version::supports_avx512dq(), "");
2811 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2812 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2813 emit_int16(0x47, (0xC0 | encode));
2814 }
2815
2816 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
2817 assert(VM_Version::supports_evex(), "");
2818 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2819 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2820 emit_int16(0x47, (0xC0 | encode));
2821 }
2822
2823 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
2824 assert(VM_Version::supports_avx512bw(), "");
2825 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2826 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2827 emit_int16(0x47, (0xC0 | encode));
2828 }
2829
2830 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
2831 assert(VM_Version::supports_avx512bw(), "");
2832 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2833 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2834 emit_int16(0x47, (0xC0 | encode));
2835 }
2836
2837 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
2838 assert(VM_Version::supports_avx512dq(), "");
2839 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2840 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2841 emit_int16(0x41, (0xC0 | encode));
2842 }
2843
2844 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
2845 assert(VM_Version::supports_evex(), "");
2846 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2847 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2848 emit_int16(0x41, (0xC0 | encode));
2849 }
2850
2851 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
2852 assert(VM_Version::supports_avx512bw(), "");
2853 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2854 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2855 emit_int16(0x41, (0xC0 | encode));
2856 }
2857
2858 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
2859 assert(VM_Version::supports_avx512bw(), "");
2860 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2861 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2862 emit_int16(0x41, (0xC0 | encode));
2863 }
2864
2865 void Assembler::knotdl(KRegister dst, KRegister src) {
2866 assert(VM_Version::supports_avx512bw(), "");
2867 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2868 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2869 emit_int16(0x44, (0xC0 | encode));
2870 }
2871
2872 void Assembler::knotql(KRegister dst, KRegister src) {
2873 assert(VM_Version::supports_avx512bw(), "");
2874 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2875 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2876 emit_int16(0x44, (0xC0 | encode));
2877 }
2878
2879 // This instruction produces ZF or CF flags
2880 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2881 assert(VM_Version::supports_avx512dq(), "");
2882 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2883 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2884 emit_int16((unsigned char)0x98, (0xC0 | encode));
2885 }
2886
2887 // This instruction produces ZF or CF flags
2888 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2889 assert(VM_Version::supports_evex(), "");
2890 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2891 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2892 emit_int16((unsigned char)0x98, (0xC0 | encode));
2893 }
2894
2895 // This instruction produces ZF or CF flags
2896 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2897 assert(VM_Version::supports_avx512bw(), "");
2898 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2899 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2900 emit_int16((unsigned char)0x98, (0xC0 | encode));
2901 }
2902
2903 // This instruction produces ZF or CF flags
2904 void Assembler::kortestql(KRegister src1, KRegister src2) {
2905 assert(VM_Version::supports_avx512bw(), "");
2906 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2907 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2908 emit_int16((unsigned char)0x98, (0xC0 | encode));
2909 }
2910
2911 // This instruction produces ZF or CF flags
2912 void Assembler::ktestql(KRegister src1, KRegister src2) {
2913 assert(VM_Version::supports_avx512bw(), "");
2914 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2915 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2916 emit_int16((unsigned char)0x99, (0xC0 | encode));
2917 }
2918
2919 void Assembler::ktestdl(KRegister src1, KRegister src2) {
2920 assert(VM_Version::supports_avx512bw(), "");
2921 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2922 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2923 emit_int16((unsigned char)0x99, (0xC0 | encode));
2924 }
2925
2926 void Assembler::ktestwl(KRegister src1, KRegister src2) {
2927 assert(VM_Version::supports_avx512dq(), "");
2928 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2929 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2930 emit_int16((unsigned char)0x99, (0xC0 | encode));
2931 }
2932
2933 void Assembler::ktestbl(KRegister src1, KRegister src2) {
2934 assert(VM_Version::supports_avx512dq(), "");
2935 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2936 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2937 emit_int16((unsigned char)0x99, (0xC0 | encode));
2938 }
2939
2940 void Assembler::ktestq(KRegister src1, KRegister src2) {
2941 assert(VM_Version::supports_avx512bw(), "");
2942 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2943 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2944 emit_int16((unsigned char)0x99, (0xC0 | encode));
2945 }
2946
2947 void Assembler::ktestd(KRegister src1, KRegister src2) {
2948 assert(VM_Version::supports_avx512bw(), "");
2949 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2950 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2951 emit_int16((unsigned char)0x99, (0xC0 | encode));
2952 }
2953
2954 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
2955 assert(VM_Version::supports_avx512dq(), "");
2956 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2957 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2958 emit_int16(0x46, (0xC0 | encode));
2959 }
2960
2961 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
2962 assert(VM_Version::supports_avx512dq(), "");
2963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2964 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2965 emit_int16(0x32, (0xC0 | encode));
2966 emit_int8(imm8);
2967 }
2968
2969 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
2970 assert(VM_Version::supports_avx512bw(), "");
2971 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2972 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2973 emit_int16(0x33, (0xC0 | encode));
2974 emit_int8(imm8);
2975 }
2976
2977
2978 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
2979 assert(VM_Version::supports_avx512dq(), "");
2980 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2981 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2982 emit_int16(0x30, (0xC0 | encode));
2983 }
2984
2985 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
2986 assert(VM_Version::supports_evex(), "");
2987 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2988 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2989 emit_int16(0x30, (0xC0 | encode));
2990 emit_int8(imm8);
2991 }
2992
2993 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
2994 assert(VM_Version::supports_avx512bw(), "");
2995 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2996 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2997 emit_int16(0x31, (0xC0 | encode));
2998 emit_int8(imm8);
2999 }
3000
3001 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
3002 assert(VM_Version::supports_avx512bw(), "");
3003 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3004 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3005 emit_int16(0x31, (0xC0 | encode));
3006 emit_int8(imm8);
3007 }
3008
3009 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
3010 assert(VM_Version::supports_avx512bw(), "");
3011 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3012 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3013 emit_int16(0x4B, (0xC0 | encode));
3014 }
3015
3016 void Assembler::movb(Address dst, int imm8) {
3017 InstructionMark im(this);
3018 prefix(dst);
3019 emit_int8((unsigned char)0xC6);
3020 emit_operand(rax, dst, 1);
3021 emit_int8(imm8);
3022 }
3023
3024
3025 void Assembler::movb(Address dst, Register src) {
3026 assert(src->has_byte_register(), "must have byte register");
3027 InstructionMark im(this);
3028 prefix(dst, src, true);
3029 emit_int8((unsigned char)0x88);
3030 emit_operand(src, dst, 0);
3031 }
3032
3033 void Assembler::movdl(XMMRegister dst, Register src) {
3034 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3035 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3036 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3037 emit_int16(0x6E, (0xC0 | encode));
3038 }
3039
3040 void Assembler::movdl(Register dst, XMMRegister src) {
3041 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3042 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3043 // swap src/dst to get correct prefix
3044 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3045 emit_int16(0x7E, (0xC0 | encode));
3046 }
3047
3048 void Assembler::movdl(XMMRegister dst, Address src) {
3049 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3050 InstructionMark im(this);
3051 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3052 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3053 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3054 emit_int8(0x6E);
3055 emit_operand(dst, src, 0);
3056 }
3057
3058 void Assembler::movdl(Address dst, XMMRegister src) {
3059 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3060 InstructionMark im(this);
3061 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3062 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3063 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3064 emit_int8(0x7E);
3065 emit_operand(src, dst, 0);
3066 }
3067
3068 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
3069 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3070 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3071 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3072 emit_int16(0x6F, (0xC0 | encode));
3073 }
3074
3075 void Assembler::movdqa(XMMRegister dst, Address src) {
3076 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3077 InstructionMark im(this);
3078 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3079 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3080 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3081 emit_int8(0x6F);
3082 emit_operand(dst, src, 0);
3083 }
3084
3085 void Assembler::movdqu(XMMRegister dst, Address src) {
3086 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3087 InstructionMark im(this);
3088 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3089 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3090 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3091 emit_int8(0x6F);
3092 emit_operand(dst, src, 0);
3093 }
3094
3095 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
3096 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3097 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3098 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3099 emit_int16(0x6F, (0xC0 | encode));
3100 }
3101
3102 void Assembler::movdqu(Address dst, XMMRegister src) {
3103 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3104 InstructionMark im(this);
3105 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3106 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3107 attributes.reset_is_clear_context();
3108 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3109 emit_int8(0x7F);
3110 emit_operand(src, dst, 0);
3111 }
3112
3113 // Move Unaligned 256bit Vector
3114 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3115 assert(UseAVX > 0, "");
3116 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3117 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3118 emit_int16(0x6F, (0xC0 | encode));
3119 }
3120
3121 void Assembler::vmovdqu(XMMRegister dst, Address src) {
3122 assert(UseAVX > 0, "");
3123 InstructionMark im(this);
3124 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3125 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3126 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3127 emit_int8(0x6F);
3128 emit_operand(dst, src, 0);
3129 }
3130
3131 void Assembler::vmovdqu(Address dst, XMMRegister src) {
3132 assert(UseAVX > 0, "");
3133 InstructionMark im(this);
3134 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3135 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3136 attributes.reset_is_clear_context();
3137 // swap src<->dst for encoding
3138 assert(src != xnoreg, "sanity");
3139 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3140 emit_int8(0x7F);
3141 emit_operand(src, dst, 0);
3142 }
3143
3144 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3145 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3146 InstructionMark im(this);
3147 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3148 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3149 emit_int8((unsigned char)0x8C);
3150 emit_operand(dst, src, 0);
3151 }
3152
3153 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3154 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3155 InstructionMark im(this);
3156 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3157 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3158 emit_int8((unsigned char)0x8C);
3159 emit_operand(dst, src, 0);
3160 }
3161
3162 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3163 assert(UseAVX > 0, "requires some form of AVX");
3164 InstructionMark im(this);
3165 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3166 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3167 emit_int8(0x2C);
3168 emit_operand(dst, src, 0);
3169 }
3170
3171 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3172 assert(UseAVX > 0, "requires some form of AVX");
3173 InstructionMark im(this);
3174 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3175 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3176 emit_int8(0x2D);
3177 emit_operand(dst, src, 0);
3178 }
3179
3180 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3181 assert(UseAVX > 0, "");
3182 InstructionMark im(this);
3183 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3184 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3185 emit_int8(0x2E);
3186 emit_operand(src, dst, 0);
3187 }
3188
3189 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3190 assert(UseAVX > 0, "");
3191 InstructionMark im(this);
3192 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3193 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3194 emit_int8(0x2F);
3195 emit_operand(src, dst, 0);
3196 }
3197
3198 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
3199 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3200 assert(VM_Version::supports_avx512vlbw(), "");
3201 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3202 attributes.set_embedded_opmask_register_specifier(mask);
3203 attributes.set_is_evex_instruction();
3204 if (merge) {
3205 attributes.reset_is_clear_context();
3206 }
3207 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3208 emit_int16(0x6F, (0xC0 | encode));
3209 }
3210
3211 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
3212 // Unmasked instruction
3213 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3214 }
3215
3216 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3217 assert(VM_Version::supports_avx512vlbw(), "");
3218 InstructionMark im(this);
3219 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3220 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3221 attributes.set_embedded_opmask_register_specifier(mask);
3222 attributes.set_is_evex_instruction();
3223 if (merge) {
3224 attributes.reset_is_clear_context();
3225 }
3226 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3227 emit_int8(0x6F);
3228 emit_operand(dst, src, 0);
3229 }
3230
3231 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
3232 // Unmasked instruction
3233 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3234 }
3235
3236 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3237 assert(VM_Version::supports_avx512vlbw(), "");
3238 assert(src != xnoreg, "sanity");
3239 InstructionMark im(this);
3240 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3241 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3242 attributes.set_embedded_opmask_register_specifier(mask);
3243 attributes.set_is_evex_instruction();
3244 if (merge) {
3245 attributes.reset_is_clear_context();
3246 }
3247 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3248 emit_int8(0x7F);
3249 emit_operand(src, dst, 0);
3250 }
3251
3252 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
3253 // Unmasked instruction
3254 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3255 }
3256
3257 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3258 assert(VM_Version::supports_avx512vlbw(), "");
3259 InstructionMark im(this);
3260 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3261 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3262 attributes.set_embedded_opmask_register_specifier(mask);
3263 attributes.set_is_evex_instruction();
3264 if (merge) {
3265 attributes.reset_is_clear_context();
3266 }
3267 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3268 emit_int8(0x6F);
3269 emit_operand(dst, src, 0);
3270 }
3271
3272 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
3273 // Unmasked instruction
3274 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3275 }
3276
3277 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3278 assert(VM_Version::supports_avx512vlbw(), "");
3279 assert(src != xnoreg, "sanity");
3280 InstructionMark im(this);
3281 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3282 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3283 attributes.set_embedded_opmask_register_specifier(mask);
3284 attributes.set_is_evex_instruction();
3285 if (merge) {
3286 attributes.reset_is_clear_context();
3287 }
3288 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3289 emit_int8(0x7F);
3290 emit_operand(src, dst, 0);
3291 }
3292
3293 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
3294 // Unmasked instruction
3295 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3296 }
3297
3298 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3299 assert(VM_Version::supports_evex(), "");
3300 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3301 attributes.set_embedded_opmask_register_specifier(mask);
3302 attributes.set_is_evex_instruction();
3303 if (merge) {
3304 attributes.reset_is_clear_context();
3305 }
3306 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3307 emit_int16(0x6F, (0xC0 | encode));
3308 }
3309
3310 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
3311 // Unmasked instruction
3312 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3313 }
3314
3315 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3316 assert(VM_Version::supports_evex(), "");
3317 InstructionMark im(this);
3318 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
3319 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3320 attributes.set_embedded_opmask_register_specifier(mask);
3321 attributes.set_is_evex_instruction();
3322 if (merge) {
3323 attributes.reset_is_clear_context();
3324 }
3325 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3326 emit_int8(0x6F);
3327 emit_operand(dst, src, 0);
3328 }
3329
3330 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
3331 // Unmasked isntruction
3332 evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
3333 }
3334
3335 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3336 assert(VM_Version::supports_evex(), "");
3337 assert(src != xnoreg, "sanity");
3338 InstructionMark im(this);
3339 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3340 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3341 attributes.set_embedded_opmask_register_specifier(mask);
3342 attributes.set_is_evex_instruction();
3343 if (merge) {
3344 attributes.reset_is_clear_context();
3345 }
3346 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3347 emit_int8(0x7F);
3348 emit_operand(src, dst, 0);
3349 }
3350
3351 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
3352 // Unmasked instruction
3353 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3354 }
3355
3356 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3357 assert(VM_Version::supports_evex(), "");
3358 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3359 attributes.set_embedded_opmask_register_specifier(mask);
3360 attributes.set_is_evex_instruction();
3361 if (merge) {
3362 attributes.reset_is_clear_context();
3363 }
3364 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3365 emit_int16(0x6F, (0xC0 | encode));
3366 }
3367
3368 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
3369 // Unmasked instruction
3370 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3371 }
3372
3373 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3374 assert(VM_Version::supports_evex(), "");
3375 InstructionMark im(this);
3376 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3377 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3378 attributes.set_embedded_opmask_register_specifier(mask);
3379 attributes.set_is_evex_instruction();
3380 if (merge) {
3381 attributes.reset_is_clear_context();
3382 }
3383 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3384 emit_int8(0x6F);
3385 emit_operand(dst, src, 0);
3386 }
3387
3388 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
3389 // Unmasked instruction
3390 evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
3391 }
3392
3393 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3394 assert(VM_Version::supports_evex(), "");
3395 assert(src != xnoreg, "sanity");
3396 InstructionMark im(this);
3397 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3398 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3399 attributes.set_embedded_opmask_register_specifier(mask);
3400 if (merge) {
3401 attributes.reset_is_clear_context();
3402 }
3403 attributes.set_is_evex_instruction();
3404 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3405 emit_int8(0x7F);
3406 emit_operand(src, dst, 0);
3407 }
3408
3409 // Uses zero extension on 64bit
3410
3411 void Assembler::movl(Register dst, int32_t imm32) {
3412 int encode = prefix_and_encode(dst->encoding());
3413 emit_int8(0xB8 | encode);
3414 emit_int32(imm32);
3415 }
3416
3417 void Assembler::movl(Register dst, Register src) {
3418 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3419 emit_int16((unsigned char)0x8B, (0xC0 | encode));
3420 }
3421
3422 void Assembler::movl(Register dst, Address src) {
3423 InstructionMark im(this);
3424 prefix(src, dst);
3425 emit_int8((unsigned char)0x8B);
3426 emit_operand(dst, src, 0);
3427 }
3428
3429 void Assembler::movl(Address dst, int32_t imm32) {
3430 InstructionMark im(this);
3431 prefix(dst);
3432 emit_int8((unsigned char)0xC7);
3433 emit_operand(rax, dst, 4);
3434 emit_int32(imm32);
3435 }
3436
3437 void Assembler::movl(Address dst, Register src) {
3438 InstructionMark im(this);
3439 prefix(dst, src);
3440 emit_int8((unsigned char)0x89);
3441 emit_operand(src, dst, 0);
3442 }
3443
3444 // New cpus require to use movsd and movss to avoid partial register stall
3445 // when loading from memory. But for old Opteron use movlpd instead of movsd.
3446 // The selection is done in MacroAssembler::movdbl() and movflt().
3447 void Assembler::movlpd(XMMRegister dst, Address src) {
3448 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3449 InstructionMark im(this);
3450 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3451 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3452 attributes.set_rex_vex_w_reverted();
3453 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3454 emit_int8(0x12);
3455 emit_operand(dst, src, 0);
3456 }
3457
3458 void Assembler::movq(XMMRegister dst, Address src) {
3459 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3460 InstructionMark im(this);
3461 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3462 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3463 attributes.set_rex_vex_w_reverted();
3464 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3465 emit_int8(0x7E);
3466 emit_operand(dst, src, 0);
3467 }
3468
3469 void Assembler::movq(Address dst, XMMRegister src) {
3470 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3471 InstructionMark im(this);
3472 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3473 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3474 attributes.set_rex_vex_w_reverted();
3475 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3476 emit_int8((unsigned char)0xD6);
3477 emit_operand(src, dst, 0);
3478 }
3479
3480 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3481 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3482 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3483 attributes.set_rex_vex_w_reverted();
3484 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3485 emit_int16((unsigned char)0xD6, (0xC0 | encode));
3486 }
3487
3488 void Assembler::movq(Register dst, XMMRegister src) {
3489 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3490 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3491 // swap src/dst to get correct prefix
3492 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3493 emit_int16(0x7E, (0xC0 | encode));
3494 }
3495
3496 void Assembler::movq(XMMRegister dst, Register src) {
3497 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3498 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3499 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3500 emit_int16(0x6E, (0xC0 | encode));
3501 }
3502
3503 void Assembler::movsbl(Register dst, Address src) { // movsxb
3504 InstructionMark im(this);
3505 prefix(src, dst);
3506 emit_int16(0x0F, (unsigned char)0xBE);
3507 emit_operand(dst, src, 0);
3508 }
3509
3510 void Assembler::movsbl(Register dst, Register src) { // movsxb
3511 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3512 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3513 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
3514 }
3515
3516 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3517 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3518 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3519 attributes.set_rex_vex_w_reverted();
3520 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3521 emit_int16(0x10, (0xC0 | encode));
3522 }
3523
3524 void Assembler::movsd(XMMRegister dst, Address src) {
3525 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3526 InstructionMark im(this);
3527 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3528 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3529 attributes.set_rex_vex_w_reverted();
3530 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3531 emit_int8(0x10);
3532 emit_operand(dst, src, 0);
3533 }
3534
3535 void Assembler::movsd(Address dst, XMMRegister src) {
3536 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3537 InstructionMark im(this);
3538 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3539 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3540 attributes.reset_is_clear_context();
3541 attributes.set_rex_vex_w_reverted();
3542 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3543 emit_int8(0x11);
3544 emit_operand(src, dst, 0);
3545 }
3546
3547 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3548 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3549 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3550 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3551 emit_int16(0x10, (0xC0 | encode));
3552 }
3553
3554 void Assembler::movss(XMMRegister dst, Address src) {
3555 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3556 InstructionMark im(this);
3557 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3558 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3559 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3560 emit_int8(0x10);
3561 emit_operand(dst, src, 0);
3562 }
3563
3564 void Assembler::movss(Address dst, XMMRegister src) {
3565 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3566 InstructionMark im(this);
3567 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3568 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3569 attributes.reset_is_clear_context();
3570 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3571 emit_int8(0x11);
3572 emit_operand(src, dst, 0);
3573 }
3574
3575 void Assembler::movswl(Register dst, Address src) { // movsxw
3576 InstructionMark im(this);
3577 prefix(src, dst);
3578 emit_int16(0x0F, (unsigned char)0xBF);
3579 emit_operand(dst, src, 0);
3580 }
3581
3582 void Assembler::movswl(Register dst, Register src) { // movsxw
3583 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3584 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
3585 }
3586
3587 void Assembler::movups(XMMRegister dst, Address src) {
3588 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3589 InstructionMark im(this);
3590 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3591 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3592 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3593 emit_int8(0x10);
3594 emit_operand(dst, src, 0);
3595 }
3596
3597 void Assembler::vmovups(XMMRegister dst, Address src, int vector_len) {
3598 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
3599 InstructionMark im(this);
3600 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3601 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3602 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3603 emit_int8(0x10);
3604 emit_operand(dst, src, 0);
3605 }
3606
3607 void Assembler::movups(Address dst, XMMRegister src) {
3608 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3609 InstructionMark im(this);
3610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3611 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3612 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3613 emit_int8(0x11);
3614 emit_operand(src, dst, 0);
3615 }
3616
3617 void Assembler::vmovups(Address dst, XMMRegister src, int vector_len) {
3618 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
3619 InstructionMark im(this);
3620 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3621 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_32bit);
3622 simd_prefix(src, xnoreg, dst, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3623 emit_int8(0x11);
3624 emit_operand(src, dst, 0);
3625 }
3626
3627 void Assembler::movw(Address dst, int imm16) {
3628 InstructionMark im(this);
3629
3630 emit_int8(0x66); // switch to 16-bit mode
3631 prefix(dst);
3632 emit_int8((unsigned char)0xC7);
3633 emit_operand(rax, dst, 2);
3634 emit_int16(imm16);
3635 }
3636
3637 void Assembler::movw(Register dst, Address src) {
3638 InstructionMark im(this);
3639 emit_int8(0x66);
3640 prefix(src, dst);
3641 emit_int8((unsigned char)0x8B);
3642 emit_operand(dst, src, 0);
3643 }
3644
3645 void Assembler::movw(Address dst, Register src) {
3646 InstructionMark im(this);
3647 emit_int8(0x66);
3648 prefix(dst, src);
3649 emit_int8((unsigned char)0x89);
3650 emit_operand(src, dst, 0);
3651 }
3652
3653 void Assembler::movzbl(Register dst, Address src) { // movzxb
3654 InstructionMark im(this);
3655 prefix(src, dst);
3656 emit_int16(0x0F, (unsigned char)0xB6);
3657 emit_operand(dst, src, 0);
3658 }
3659
3660 void Assembler::movzbl(Register dst, Register src) { // movzxb
3661 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3662 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3663 emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
3664 }
3665
3666 void Assembler::movzwl(Register dst, Address src) { // movzxw
3667 InstructionMark im(this);
3668 prefix(src, dst);
3669 emit_int16(0x0F, (unsigned char)0xB7);
3670 emit_operand(dst, src, 0);
3671 }
3672
3673 void Assembler::movzwl(Register dst, Register src) { // movzxw
3674 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3675 emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
3676 }
3677
3678 void Assembler::mull(Address src) {
3679 InstructionMark im(this);
3680 prefix(src);
3681 emit_int8((unsigned char)0xF7);
3682 emit_operand(rsp, src, 0);
3683 }
3684
3685 void Assembler::mull(Register src) {
3686 int encode = prefix_and_encode(src->encoding());
3687 emit_int16((unsigned char)0xF7, (0xE0 | encode));
3688 }
3689
3690 void Assembler::mulsd(XMMRegister dst, Address src) {
3691 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3692 InstructionMark im(this);
3693 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3694 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3695 attributes.set_rex_vex_w_reverted();
3696 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3697 emit_int8(0x59);
3698 emit_operand(dst, src, 0);
3699 }
3700
3701 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3702 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3703 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3704 attributes.set_rex_vex_w_reverted();
3705 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3706 emit_int16(0x59, (0xC0 | encode));
3707 }
3708
3709 void Assembler::mulss(XMMRegister dst, Address src) {
3710 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3711 InstructionMark im(this);
3712 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3713 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3714 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3715 emit_int8(0x59);
3716 emit_operand(dst, src, 0);
3717 }
3718
3719 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3720 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3721 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3722 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3723 emit_int16(0x59, (0xC0 | encode));
3724 }
3725
3726 void Assembler::negl(Register dst) {
3727 int encode = prefix_and_encode(dst->encoding());
3728 emit_int16((unsigned char)0xF7, (0xD8 | encode));
3729 }
3730
3731 void Assembler::negl(Address dst) {
3732 InstructionMark im(this);
3733 prefix(dst);
3734 emit_int8((unsigned char)0xF7);
3735 emit_operand(as_Register(3), dst, 0);
3736 }
3737
3738 void Assembler::nop(int i) {
3739 #ifdef ASSERT
3740 assert(i > 0, " ");
3741 // The fancy nops aren't currently recognized by debuggers making it a
3742 // pain to disassemble code while debugging. If asserts are on clearly
3743 // speed is not an issue so simply use the single byte traditional nop
3744 // to do alignment.
3745
3746 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3747 return;
3748
3749 #endif // ASSERT
3750
3751 if (UseAddressNop && VM_Version::is_intel()) {
3752 //
3753 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3754 // 1: 0x90
3755 // 2: 0x66 0x90
3756 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3757 // 4: 0x0F 0x1F 0x40 0x00
3758 // 5: 0x0F 0x1F 0x44 0x00 0x00
3759 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3760 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3761 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3762 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3763 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3764 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3765
3766 // The rest coding is Intel specific - don't use consecutive address nops
3767
3768 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3769 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3770 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3771 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3772
3773 while(i >= 15) {
3774 // For Intel don't generate consecutive address nops (mix with regular nops)
3775 i -= 15;
3776 emit_int24(0x66, 0x66, 0x66);
3777 addr_nop_8();
3778 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3779 }
3780 switch (i) {
3781 case 14:
3782 emit_int8(0x66); // size prefix
3783 case 13:
3784 emit_int8(0x66); // size prefix
3785 case 12:
3786 addr_nop_8();
3787 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3788 break;
3789 case 11:
3790 emit_int8(0x66); // size prefix
3791 case 10:
3792 emit_int8(0x66); // size prefix
3793 case 9:
3794 emit_int8(0x66); // size prefix
3795 case 8:
3796 addr_nop_8();
3797 break;
3798 case 7:
3799 addr_nop_7();
3800 break;
3801 case 6:
3802 emit_int8(0x66); // size prefix
3803 case 5:
3804 addr_nop_5();
3805 break;
3806 case 4:
3807 addr_nop_4();
3808 break;
3809 case 3:
3810 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3811 emit_int8(0x66); // size prefix
3812 case 2:
3813 emit_int8(0x66); // size prefix
3814 case 1:
3815 emit_int8((unsigned char)0x90);
3816 // nop
3817 break;
3818 default:
3819 assert(i == 0, " ");
3820 }
3821 return;
3822 }
3823 if (UseAddressNop && VM_Version::is_amd_family()) {
3824 //
3825 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3826 // 1: 0x90
3827 // 2: 0x66 0x90
3828 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3829 // 4: 0x0F 0x1F 0x40 0x00
3830 // 5: 0x0F 0x1F 0x44 0x00 0x00
3831 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3832 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3833 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3834 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3835 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3836 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3837
3838 // The rest coding is AMD specific - use consecutive address nops
3839
3840 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3841 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3842 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3843 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3844 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3845 // Size prefixes (0x66) are added for larger sizes
3846
3847 while(i >= 22) {
3848 i -= 11;
3849 emit_int24(0x66, 0x66, 0x66);
3850 addr_nop_8();
3851 }
3852 // Generate first nop for size between 21-12
3853 switch (i) {
3854 case 21:
3855 i -= 1;
3856 emit_int8(0x66); // size prefix
3857 case 20:
3858 case 19:
3859 i -= 1;
3860 emit_int8(0x66); // size prefix
3861 case 18:
3862 case 17:
3863 i -= 1;
3864 emit_int8(0x66); // size prefix
3865 case 16:
3866 case 15:
3867 i -= 8;
3868 addr_nop_8();
3869 break;
3870 case 14:
3871 case 13:
3872 i -= 7;
3873 addr_nop_7();
3874 break;
3875 case 12:
3876 i -= 6;
3877 emit_int8(0x66); // size prefix
3878 addr_nop_5();
3879 break;
3880 default:
3881 assert(i < 12, " ");
3882 }
3883
3884 // Generate second nop for size between 11-1
3885 switch (i) {
3886 case 11:
3887 emit_int8(0x66); // size prefix
3888 case 10:
3889 emit_int8(0x66); // size prefix
3890 case 9:
3891 emit_int8(0x66); // size prefix
3892 case 8:
3893 addr_nop_8();
3894 break;
3895 case 7:
3896 addr_nop_7();
3897 break;
3898 case 6:
3899 emit_int8(0x66); // size prefix
3900 case 5:
3901 addr_nop_5();
3902 break;
3903 case 4:
3904 addr_nop_4();
3905 break;
3906 case 3:
3907 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3908 emit_int8(0x66); // size prefix
3909 case 2:
3910 emit_int8(0x66); // size prefix
3911 case 1:
3912 emit_int8((unsigned char)0x90);
3913 // nop
3914 break;
3915 default:
3916 assert(i == 0, " ");
3917 }
3918 return;
3919 }
3920
3921 if (UseAddressNop && VM_Version::is_zx()) {
3922 //
3923 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3924 // 1: 0x90
3925 // 2: 0x66 0x90
3926 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3927 // 4: 0x0F 0x1F 0x40 0x00
3928 // 5: 0x0F 0x1F 0x44 0x00 0x00
3929 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3930 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3931 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3932 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3933 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3934 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3935
3936 // The rest coding is ZX specific - don't use consecutive address nops
3937
3938 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3939 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3940 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3941 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3942
3943 while (i >= 15) {
3944 // For ZX don't generate consecutive address nops (mix with regular nops)
3945 i -= 15;
3946 emit_int24(0x66, 0x66, 0x66);
3947 addr_nop_8();
3948 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3949 }
3950 switch (i) {
3951 case 14:
3952 emit_int8(0x66); // size prefix
3953 case 13:
3954 emit_int8(0x66); // size prefix
3955 case 12:
3956 addr_nop_8();
3957 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3958 break;
3959 case 11:
3960 emit_int8(0x66); // size prefix
3961 case 10:
3962 emit_int8(0x66); // size prefix
3963 case 9:
3964 emit_int8(0x66); // size prefix
3965 case 8:
3966 addr_nop_8();
3967 break;
3968 case 7:
3969 addr_nop_7();
3970 break;
3971 case 6:
3972 emit_int8(0x66); // size prefix
3973 case 5:
3974 addr_nop_5();
3975 break;
3976 case 4:
3977 addr_nop_4();
3978 break;
3979 case 3:
3980 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3981 emit_int8(0x66); // size prefix
3982 case 2:
3983 emit_int8(0x66); // size prefix
3984 case 1:
3985 emit_int8((unsigned char)0x90);
3986 // nop
3987 break;
3988 default:
3989 assert(i == 0, " ");
3990 }
3991 return;
3992 }
3993
3994 // Using nops with size prefixes "0x66 0x90".
3995 // From AMD Optimization Guide:
3996 // 1: 0x90
3997 // 2: 0x66 0x90
3998 // 3: 0x66 0x66 0x90
3999 // 4: 0x66 0x66 0x66 0x90
4000 // 5: 0x66 0x66 0x90 0x66 0x90
4001 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
4002 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
4003 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
4004 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
4005 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
4006 //
4007 while (i > 12) {
4008 i -= 4;
4009 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
4010 }
4011 // 1 - 12 nops
4012 if (i > 8) {
4013 if (i > 9) {
4014 i -= 1;
4015 emit_int8(0x66);
4016 }
4017 i -= 3;
4018 emit_int24(0x66, 0x66, (unsigned char)0x90);
4019 }
4020 // 1 - 8 nops
4021 if (i > 4) {
4022 if (i > 6) {
4023 i -= 1;
4024 emit_int8(0x66);
4025 }
4026 i -= 3;
4027 emit_int24(0x66, 0x66, (unsigned char)0x90);
4028 }
4029 switch (i) {
4030 case 4:
4031 emit_int8(0x66);
4032 case 3:
4033 emit_int8(0x66);
4034 case 2:
4035 emit_int8(0x66);
4036 case 1:
4037 emit_int8((unsigned char)0x90);
4038 break;
4039 default:
4040 assert(i == 0, " ");
4041 }
4042 }
4043
4044 void Assembler::notl(Register dst) {
4045 int encode = prefix_and_encode(dst->encoding());
4046 emit_int16((unsigned char)0xF7, (0xD0 | encode));
4047 }
4048
4049 void Assembler::orw(Register dst, Register src) {
4050 (void)prefix_and_encode(dst->encoding(), src->encoding());
4051 emit_arith(0x0B, 0xC0, dst, src);
4052 }
4053
4054 void Assembler::orl(Address dst, int32_t imm32) {
4055 InstructionMark im(this);
4056 prefix(dst);
4057 emit_arith_operand(0x81, rcx, dst, imm32);
4058 }
4059
4060 void Assembler::orl(Register dst, int32_t imm32) {
4061 prefix(dst);
4062 emit_arith(0x81, 0xC8, dst, imm32);
4063 }
4064
4065 void Assembler::orl(Register dst, Address src) {
4066 InstructionMark im(this);
4067 prefix(src, dst);
4068 emit_int8(0x0B);
4069 emit_operand(dst, src, 0);
4070 }
4071
4072 void Assembler::orl(Register dst, Register src) {
4073 (void) prefix_and_encode(dst->encoding(), src->encoding());
4074 emit_arith(0x0B, 0xC0, dst, src);
4075 }
4076
4077 void Assembler::orl(Address dst, Register src) {
4078 InstructionMark im(this);
4079 prefix(dst, src);
4080 emit_int8(0x09);
4081 emit_operand(src, dst, 0);
4082 }
4083
4084 void Assembler::orb(Address dst, int imm8) {
4085 InstructionMark im(this);
4086 prefix(dst);
4087 emit_int8((unsigned char)0x80);
4088 emit_operand(rcx, dst, 1);
4089 emit_int8(imm8);
4090 }
4091
4092 void Assembler::orb(Address dst, Register src) {
4093 InstructionMark im(this);
4094 prefix(dst, src, true);
4095 emit_int8(0x08);
4096 emit_operand(src, dst, 0);
4097 }
4098
4099 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
4100 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4101 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4102 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4103 emit_int16(0x63, (0xC0 | encode));
4104 }
4105
4106 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4107 assert(UseAVX > 0, "some form of AVX must be enabled");
4108 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4109 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4110 emit_int16(0x63, (0xC0 | encode));
4111 }
4112
4113 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
4114 assert(VM_Version::supports_sse2(), "");
4115 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4116 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4117 emit_int16(0x6B, (0xC0 | encode));
4118 }
4119
4120 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4121 assert(UseAVX > 0, "some form of AVX must be enabled");
4122 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4123 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4124 emit_int16(0x6B, (0xC0 | encode));
4125 }
4126
4127 void Assembler::packuswb(XMMRegister dst, Address src) {
4128 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4129 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4130 InstructionMark im(this);
4131 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4132 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4133 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4134 emit_int8(0x67);
4135 emit_operand(dst, src, 0);
4136 }
4137
4138 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
4139 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4140 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4141 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4142 emit_int16(0x67, (0xC0 | encode));
4143 }
4144
4145 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4146 assert(UseAVX > 0, "some form of AVX must be enabled");
4147 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4148 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4149 emit_int16(0x67, (0xC0 | encode));
4150 }
4151
4152 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
4153 assert(VM_Version::supports_sse4_1(), "");
4154 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4155 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4156 emit_int16(0x2B, (0xC0 | encode));
4157 }
4158
4159 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4160 assert(UseAVX > 0, "some form of AVX must be enabled");
4161 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4162 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4163 emit_int16(0x2B, (0xC0 | encode));
4164 }
4165
4166 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4167 assert(VM_Version::supports_avx2(), "");
4168 assert(vector_len != AVX_128bit, "");
4169 // VEX.256.66.0F3A.W1 00 /r ib
4170 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4171 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4172 emit_int24(0x00, (0xC0 | encode), imm8);
4173 }
4174
4175 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4176 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
4177 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported");
4178 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4179 attributes.set_is_evex_instruction();
4180 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4181 emit_int16(0x36, (0xC0 | encode));
4182 }
4183
4184 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4185 assert(VM_Version::supports_avx512_vbmi(), "");
4186 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4187 attributes.set_is_evex_instruction();
4188 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4189 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4190 }
4191
4192 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4193 assert(VM_Version::supports_avx512_vbmi(), "");
4194 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4195 attributes.set_is_evex_instruction();
4196 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4197 emit_int8((unsigned char)0x8D);
4198 emit_operand(dst, src, 0);
4199 }
4200
4201 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4202 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
4203 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
4204 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
4205 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4206 attributes.set_is_evex_instruction();
4207 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4208 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4209 }
4210
4211 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4212 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4213 // VEX.NDS.256.66.0F38.W0 36 /r
4214 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4215 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4216 emit_int16(0x36, (0xC0 | encode));
4217 }
4218
4219 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4220 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4221 // VEX.NDS.256.66.0F38.W0 36 /r
4222 InstructionMark im(this);
4223 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4224 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4225 emit_int8(0x36);
4226 emit_operand(dst, src, 0);
4227 }
4228
4229 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4230 assert(VM_Version::supports_avx2(), "");
4231 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4232 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4233 emit_int24(0x46, (0xC0 | encode), imm8);
4234 }
4235
4236 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4237 assert(VM_Version::supports_avx(), "");
4238 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4240 emit_int24(0x06, (0xC0 | encode), imm8);
4241 }
4242
4243 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4244 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4245 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4246 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4247 emit_int24(0x04, (0xC0 | encode), imm8);
4248 }
4249
4250 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4251 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4252 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
4253 attributes.set_rex_vex_w_reverted();
4254 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4255 emit_int24(0x05, (0xC0 | encode), imm8);
4256 }
4257
4258 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4259 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4260 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
4261 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4262 emit_int24(0x01, (0xC0 | encode), imm8);
4263 }
4264
4265 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4266 assert(VM_Version::supports_evex(), "");
4267 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4268 attributes.set_is_evex_instruction();
4269 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4270 emit_int16(0x76, (0xC0 | encode));
4271 }
4272
4273 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4274 assert(VM_Version::supports_avx512_vbmi(), "");
4275 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4276 attributes.set_is_evex_instruction();
4277 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4278 emit_int16(0x7D, (0xC0 | encode));
4279 }
4280
4281 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
4282 assert(VM_Version::supports_avx512_vbmi(), "");
4283 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4284 attributes.set_is_evex_instruction();
4285 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4286 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
4287 }
4288
4289 void Assembler::pause() {
4290 emit_int16((unsigned char)0xF3, (unsigned char)0x90);
4291 }
4292
4293 void Assembler::ud2() {
4294 emit_int16(0x0F, 0x0B);
4295 }
4296
4297 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
4298 assert(VM_Version::supports_sse4_2(), "");
4299 InstructionMark im(this);
4300 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4301 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4302 emit_int8(0x61);
4303 emit_operand(dst, src, 1);
4304 emit_int8(imm8);
4305 }
4306
4307 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
4308 assert(VM_Version::supports_sse4_2(), "");
4309 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4310 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4311 emit_int24(0x61, (0xC0 | encode), imm8);
4312 }
4313
4314 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4315 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
4316 assert(VM_Version::supports_sse2(), "");
4317 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4318 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4319 emit_int16(0x74, (0xC0 | encode));
4320 }
4321
4322 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4323 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4324 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4325 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4326 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4327 emit_int16(cond_encoding, (0xC0 | encode));
4328 }
4329
4330 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4331 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4332 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4333 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4334 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4335 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4336 emit_int16(0x74, (0xC0 | encode));
4337 }
4338
4339 // In this context, kdst is written the mask used to process the equal components
4340 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4341 assert(VM_Version::supports_avx512bw(), "");
4342 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4343 attributes.set_is_evex_instruction();
4344 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4345 emit_int16(0x74, (0xC0 | encode));
4346 }
4347
4348 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4349 assert(VM_Version::supports_avx512vlbw(), "");
4350 InstructionMark im(this);
4351 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4352 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4353 attributes.set_is_evex_instruction();
4354 int dst_enc = kdst->encoding();
4355 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4356 emit_int8(0x64);
4357 emit_operand(as_Register(dst_enc), src, 0);
4358 }
4359
4360 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4361 assert(VM_Version::supports_avx512vlbw(), "");
4362 InstructionMark im(this);
4363 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4364 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4365 attributes.reset_is_clear_context();
4366 attributes.set_embedded_opmask_register_specifier(mask);
4367 attributes.set_is_evex_instruction();
4368 int dst_enc = kdst->encoding();
4369 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4370 emit_int8(0x64);
4371 emit_operand(as_Register(dst_enc), src, 0);
4372 }
4373
4374 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4375 assert(VM_Version::supports_avx512vlbw(), "");
4376 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4377 attributes.set_is_evex_instruction();
4378 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4379 emit_int24(0x3E, (0xC0 | encode), vcc);
4380 }
4381
4382 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
4383 assert(VM_Version::supports_avx512vlbw(), "");
4384 InstructionMark im(this);
4385 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4386 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4387 attributes.set_is_evex_instruction();
4388 int dst_enc = kdst->encoding();
4389 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4390 emit_int8(0x3E);
4391 emit_operand(as_Register(dst_enc), src, 1);
4392 emit_int8(vcc);
4393 }
4394
4395 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4396 assert(VM_Version::supports_avx512bw(), "");
4397 InstructionMark im(this);
4398 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4399 attributes.set_is_evex_instruction();
4400 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4401 int dst_enc = kdst->encoding();
4402 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4403 emit_int8(0x74);
4404 emit_operand(as_Register(dst_enc), src, 0);
4405 }
4406
4407 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4408 assert(VM_Version::supports_avx512vlbw(), "");
4409 InstructionMark im(this);
4410 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4411 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4412 attributes.reset_is_clear_context();
4413 attributes.set_embedded_opmask_register_specifier(mask);
4414 attributes.set_is_evex_instruction();
4415 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4416 emit_int8(0x74);
4417 emit_operand(as_Register(kdst->encoding()), src, 0);
4418 }
4419
4420 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4421 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
4422 assert(VM_Version::supports_sse2(), "");
4423 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4424 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4425 emit_int16(0x75, (0xC0 | encode));
4426 }
4427
4428 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4429 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4430 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4431 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4432 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4433 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4434 emit_int16(0x75, (0xC0 | encode));
4435 }
4436
4437 // In this context, kdst is written the mask used to process the equal components
4438 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4439 assert(VM_Version::supports_avx512bw(), "");
4440 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4441 attributes.set_is_evex_instruction();
4442 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4443 emit_int16(0x75, (0xC0 | encode));
4444 }
4445
4446 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4447 assert(VM_Version::supports_avx512bw(), "");
4448 InstructionMark im(this);
4449 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4450 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4451 attributes.set_is_evex_instruction();
4452 int dst_enc = kdst->encoding();
4453 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4454 emit_int8(0x75);
4455 emit_operand(as_Register(dst_enc), src, 0);
4456 }
4457
4458 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4459 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
4460 assert(VM_Version::supports_sse2(), "");
4461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4462 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4463 emit_int16(0x76, (0xC0 | encode));
4464 }
4465
4466 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4467 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4468 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4469 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4470 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4471 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4472 emit_int16(0x76, (0xC0 | encode));
4473 }
4474
4475 // In this context, kdst is written the mask used to process the equal components
4476 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4477 assert(VM_Version::supports_evex(), "");
4478 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4479 attributes.set_is_evex_instruction();
4480 attributes.reset_is_clear_context();
4481 attributes.set_embedded_opmask_register_specifier(mask);
4482 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4483 emit_int16(0x76, (0xC0 | encode));
4484 }
4485
4486 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4487 assert(VM_Version::supports_evex(), "");
4488 InstructionMark im(this);
4489 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4490 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4491 attributes.set_is_evex_instruction();
4492 attributes.reset_is_clear_context();
4493 attributes.set_embedded_opmask_register_specifier(mask);
4494 int dst_enc = kdst->encoding();
4495 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4496 emit_int8(0x76);
4497 emit_operand(as_Register(dst_enc), src, 0);
4498 }
4499
4500 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4501 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
4502 assert(VM_Version::supports_sse4_1(), "");
4503 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4504 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4505 emit_int16(0x29, (0xC0 | encode));
4506 }
4507
4508 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4509 assert(VM_Version::supports_evex(), "");
4510 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4511 attributes.set_is_evex_instruction();
4512 attributes.reset_is_clear_context();
4513 attributes.set_embedded_opmask_register_specifier(mask);
4514 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4515 emit_int16(0x29, (0xC0 | encode));
4516 }
4517
4518 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4519 assert(VM_Version::supports_avx(), "");
4520 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4521 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4522 emit_int16(cond_encoding, (0xC0 | encode));
4523 }
4524
4525 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4526 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4527 assert(VM_Version::supports_avx(), "");
4528 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4530 emit_int16(0x29, (0xC0 | encode));
4531 }
4532
4533 // In this context, kdst is written the mask used to process the equal components
4534 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4535 assert(VM_Version::supports_evex(), "");
4536 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4537 attributes.reset_is_clear_context();
4538 attributes.set_is_evex_instruction();
4539 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4540 emit_int16(0x29, (0xC0 | encode));
4541 }
4542
4543 // In this context, kdst is written the mask used to process the equal components
4544 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4545 assert(VM_Version::supports_evex(), "");
4546 InstructionMark im(this);
4547 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4548 attributes.reset_is_clear_context();
4549 attributes.set_is_evex_instruction();
4550 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4551 int dst_enc = kdst->encoding();
4552 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4553 emit_int8(0x29);
4554 emit_operand(as_Register(dst_enc), src, 0);
4555 }
4556
4557 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
4558 assert(VM_Version::supports_sse4_1(), "");
4559 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4560 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4561 emit_int16(0x37, (0xC0 | encode));
4562 }
4563
4564 void Assembler::pmovmskb(Register dst, XMMRegister src) {
4565 assert(VM_Version::supports_sse2(), "");
4566 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4567 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4568 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4569 }
4570
4571 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
4572 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
4573 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), "");
4574 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4575 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4576 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4577 }
4578
4579 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
4580 assert(VM_Version::supports_avx(), "");
4581 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4582 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4583 emit_int16(0x50, (0xC0 | encode));
4584 }
4585
4586 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
4587 assert(VM_Version::supports_avx(), "");
4588 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4589 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4590 emit_int16(0x50, (0xC0 | encode));
4591 }
4592
4593
4594 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
4595 assert(VM_Version::supports_sse4_1(), "");
4596 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4597 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4598 emit_int24(0x16, (0xC0 | encode), imm8);
4599 }
4600
4601 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
4602 assert(VM_Version::supports_sse4_1(), "");
4603 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4604 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4605 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4606 emit_int8(0x16);
4607 emit_operand(src, dst, 1);
4608 emit_int8(imm8);
4609 }
4610
4611 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
4612 assert(VM_Version::supports_sse4_1(), "");
4613 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4614 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4615 emit_int24(0x16, (0xC0 | encode), imm8);
4616 }
4617
4618 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
4619 assert(VM_Version::supports_sse4_1(), "");
4620 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4621 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4622 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4623 emit_int8(0x16);
4624 emit_operand(src, dst, 1);
4625 emit_int8(imm8);
4626 }
4627
4628 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
4629 assert(VM_Version::supports_sse2(), "");
4630 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4631 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4632 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
4633 }
4634
4635 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
4636 assert(VM_Version::supports_sse4_1(), "");
4637 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4638 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4639 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4640 emit_int8(0x15);
4641 emit_operand(src, dst, 1);
4642 emit_int8(imm8);
4643 }
4644
4645 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
4646 assert(VM_Version::supports_sse4_1(), "");
4647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4648 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4649 emit_int24(0x14, (0xC0 | encode), imm8);
4650 }
4651
4652 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
4653 assert(VM_Version::supports_sse4_1(), "");
4654 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4655 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4656 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4657 emit_int8(0x14);
4658 emit_operand(src, dst, 1);
4659 emit_int8(imm8);
4660 }
4661
4662 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
4663 assert(VM_Version::supports_sse4_1(), "");
4664 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4665 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4666 emit_int24(0x22, (0xC0 | encode), imm8);
4667 }
4668
4669 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
4670 assert(VM_Version::supports_sse4_1(), "");
4671 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4672 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4673 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4674 emit_int8(0x22);
4675 emit_operand(dst, src, 1);
4676 emit_int8(imm8);
4677 }
4678
4679 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4680 assert(VM_Version::supports_avx(), "");
4681 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4683 emit_int24(0x22, (0xC0 | encode), imm8);
4684 }
4685
4686 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
4687 assert(VM_Version::supports_sse4_1(), "");
4688 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4689 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4690 emit_int24(0x22, (0xC0 | encode), imm8);
4691 }
4692
4693 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
4694 assert(VM_Version::supports_sse4_1(), "");
4695 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4696 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4697 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4698 emit_int8(0x22);
4699 emit_operand(dst, src, 1);
4700 emit_int8(imm8);
4701 }
4702
4703 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4704 assert(VM_Version::supports_avx(), "");
4705 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4706 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4707 emit_int24(0x22, (0xC0 | encode), imm8);
4708 }
4709
4710 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
4711 assert(VM_Version::supports_sse2(), "");
4712 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4713 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4714 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4715 }
4716
4717 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
4718 assert(VM_Version::supports_sse2(), "");
4719 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4720 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4721 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4722 emit_int8((unsigned char)0xC4);
4723 emit_operand(dst, src, 1);
4724 emit_int8(imm8);
4725 }
4726
4727 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4728 assert(VM_Version::supports_avx(), "");
4729 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4730 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4731 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4732 }
4733
4734 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
4735 assert(VM_Version::supports_sse4_1(), "");
4736 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4737 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4738 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4739 emit_int8(0x20);
4740 emit_operand(dst, src, 1);
4741 emit_int8(imm8);
4742 }
4743
4744 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
4745 assert(VM_Version::supports_sse4_1(), "");
4746 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4747 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4748 emit_int24(0x20, (0xC0 | encode), imm8);
4749 }
4750
4751 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4752 assert(VM_Version::supports_avx(), "");
4753 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4754 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4755 emit_int24(0x20, (0xC0 | encode), imm8);
4756 }
4757
4758 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
4759 assert(VM_Version::supports_sse4_1(), "");
4760 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4761 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4762 emit_int24(0x21, (0xC0 | encode), imm8);
4763 }
4764
4765 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4766 assert(VM_Version::supports_avx(), "");
4767 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4768 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4769 emit_int24(0x21, (0xC0 | encode), imm8);
4770 }
4771
4772 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
4773 assert(VM_Version::supports_sse4_1(), "");
4774 InstructionMark im(this);
4775 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4776 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4777 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4778 emit_int8(0x30);
4779 emit_operand(dst, src, 0);
4780 }
4781
4782 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
4783 assert(VM_Version::supports_sse4_1(), "");
4784 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4785 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4786 emit_int16(0x30, (0xC0 | encode));
4787 }
4788
4789 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
4790 assert(VM_Version::supports_sse4_1(), "");
4791 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4792 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4793 emit_int16(0x20, (0xC0 | encode));
4794 }
4795
4796 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
4797 assert(VM_Version::supports_sse4_1(), "");
4798 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4799 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4800 emit_int16(0x35, (0xC0 | encode));
4801 }
4802
4803 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
4804 assert(VM_Version::supports_sse4_1(), "");
4805 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4806 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4807 emit_int16(0x21, (0xC0 | encode));
4808 }
4809
4810 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
4811 assert(VM_Version::supports_sse4_1(), "");
4812 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4813 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4814 emit_int16(0x31, (0xC0 | encode));
4815 }
4816
4817 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
4818 assert(VM_Version::supports_sse4_1(), "");
4819 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4820 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4821 emit_int16(0x22, (0xC0 | encode));
4822 }
4823
4824 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
4825 assert(VM_Version::supports_sse4_1(), "");
4826 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4827 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4828 emit_int16(0x23, (0xC0 | encode));
4829 }
4830
4831 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4832 assert(VM_Version::supports_avx(), "");
4833 InstructionMark im(this);
4834 assert(dst != xnoreg, "sanity");
4835 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4836 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4837 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4838 emit_int8(0x30);
4839 emit_operand(dst, src, 0);
4840 }
4841
4842 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4843 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4844 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4845 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4846 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4847 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4848 emit_int16(0x30, (unsigned char) (0xC0 | encode));
4849 }
4850
4851 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4852 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4853 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4854 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4855 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4856 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4857 emit_int16(0x20, (0xC0 | encode));
4858 }
4859
4860 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
4861 assert(VM_Version::supports_avx512vlbw(), "");
4862 assert(dst != xnoreg, "sanity");
4863 InstructionMark im(this);
4864 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4865 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4866 attributes.set_embedded_opmask_register_specifier(mask);
4867 attributes.set_is_evex_instruction();
4868 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4869 emit_int8(0x30);
4870 emit_operand(dst, src, 0);
4871 }
4872
4873 void Assembler::evpmovzxbd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
4874 assert(VM_Version::supports_avx512vl(), "");
4875 assert(dst != xnoreg, "sanity");
4876 InstructionMark im(this);
4877 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4878 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4879 attributes.set_embedded_opmask_register_specifier(mask);
4880 attributes.set_is_evex_instruction();
4881 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4882 emit_int8(0x31);
4883 emit_operand(dst, src, 0);
4884 }
4885
4886 void Assembler::evpmovzxbd(XMMRegister dst, Address src, int vector_len) {
4887 evpmovzxbd(dst, k0, src, vector_len);
4888 }
4889
4890 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
4891 assert(VM_Version::supports_evex(), "");
4892 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
4893 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4894 attributes.set_is_evex_instruction();
4895 attributes.set_embedded_opmask_register_specifier(mask);
4896 if (merge) {
4897 attributes.reset_is_clear_context();
4898 }
4899 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4900 emit_int16((unsigned char)0xDB, (0xC0 | encode));
4901 }
4902
4903 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4904 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4905 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4906 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4907 emit_int16(0x35, (0xC0 | encode));
4908 }
4909
4910 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4911 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4912 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4913 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4914 emit_int16(0x31, (0xC0 | encode));
4915 }
4916
4917 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4918 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4919 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4920 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4921 emit_int16(0x32, (0xC0 | encode));
4922 }
4923
4924 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4925 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4926 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4927 VM_Version::supports_evex(), "");
4928 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4929 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4930 emit_int16(0x21, (0xC0 | encode));
4931 }
4932
4933 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4934 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4935 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4936 VM_Version::supports_evex(), "");
4937 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4938 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4939 emit_int16(0x22, (0xC0 | encode));
4940 }
4941
4942 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
4943 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4944 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4945 VM_Version::supports_evex(), "");
4946 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4947 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4948 emit_int16(0x23, (0xC0 | encode));
4949 }
4950
4951 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4952 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4953 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4954 VM_Version::supports_evex(), "");
4955 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4956 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4957 emit_int16(0x24, (0xC0 | encode));
4958 }
4959
4960 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4961 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4962 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4963 VM_Version::supports_evex(), "");
4964 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4965 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4966 emit_int16(0x25, (0xC0 | encode));
4967 }
4968
4969 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
4970 assert(VM_Version::supports_avx512vlbw(), "");
4971 assert(src != xnoreg, "sanity");
4972 InstructionMark im(this);
4973 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4974 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4975 attributes.set_is_evex_instruction();
4976 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4977 emit_int8(0x30);
4978 emit_operand(src, dst, 0);
4979 }
4980
4981 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
4982 assert(VM_Version::supports_avx512vlbw(), "");
4983 assert(src != xnoreg, "sanity");
4984 InstructionMark im(this);
4985 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4986 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4987 attributes.reset_is_clear_context();
4988 attributes.set_embedded_opmask_register_specifier(mask);
4989 attributes.set_is_evex_instruction();
4990 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4991 emit_int8(0x30);
4992 emit_operand(src, dst, 0);
4993 }
4994
4995 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
4996 assert(VM_Version::supports_evex(), "");
4997 assert(src != xnoreg, "sanity");
4998 InstructionMark im(this);
4999 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5000 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
5001 attributes.set_is_evex_instruction();
5002 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
5003 emit_int8(0x31);
5004 emit_operand(src, dst, 0);
5005 }
5006
5007 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
5008 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5009 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5010 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
5011 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5012 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5013 emit_int16(0x33, (0xC0 | encode));
5014 }
5015
5016 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
5017 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5018 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5019 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
5020 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5021 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5022 emit_int16(0x34, (0xC0 | encode));
5023 }
5024
5025 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
5026 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5027 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5028 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5029 emit_int16((unsigned char)0xF5, (0xC0 | encode));
5030 }
5031
5032 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5033 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5034 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5035 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
5036 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5037 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5038 emit_int16((unsigned char)0xF5, (0xC0 | encode));
5039 }
5040
5041 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5042 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5043 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5044 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5045 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5046 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5047 emit_int16(0x04, (0xC0 | encode));
5048 }
5049
5050 void Assembler::evpmadd52luq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5051 evpmadd52luq(dst, k0, src1, src2, false, vector_len);
5052 }
5053
5054 void Assembler::evpmadd52luq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5055 assert(VM_Version::supports_avx512ifma(), "");
5056 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5057 attributes.set_is_evex_instruction();
5058 attributes.set_embedded_opmask_register_specifier(mask);
5059 if (merge) {
5060 attributes.reset_is_clear_context();
5061 }
5062
5063 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5064 emit_int16((unsigned char)0xB4, (0xC0 | encode));
5065 }
5066
5067 void Assembler::evpmadd52huq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5068 evpmadd52huq(dst, k0, src1, src2, false, vector_len);
5069 }
5070
5071 void Assembler::evpmadd52huq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5072 assert(VM_Version::supports_avx512ifma(), "");
5073 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5074 attributes.set_is_evex_instruction();
5075 attributes.set_embedded_opmask_register_specifier(mask);
5076 if (merge) {
5077 attributes.reset_is_clear_context();
5078 }
5079
5080 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5081 emit_int16((unsigned char)0xB5, (0xC0 | encode));
5082 }
5083
5084 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5085 assert(VM_Version::supports_evex(), "");
5086 assert(VM_Version::supports_avx512_vnni(), "must support vnni");
5087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5088 attributes.set_is_evex_instruction();
5089 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5090 emit_int16(0x52, (0xC0 | encode));
5091 }
5092
5093 // generic
5094 void Assembler::pop(Register dst) {
5095 int encode = prefix_and_encode(dst->encoding());
5096 emit_int8(0x58 | encode);
5097 }
5098
5099 void Assembler::popcntl(Register dst, Address src) {
5100 assert(VM_Version::supports_popcnt(), "must support");
5101 InstructionMark im(this);
5102 emit_int8((unsigned char)0xF3);
5103 prefix(src, dst);
5104 emit_int16(0x0F, (unsigned char)0xB8);
5105 emit_operand(dst, src, 0);
5106 }
5107
5108 void Assembler::popcntl(Register dst, Register src) {
5109 assert(VM_Version::supports_popcnt(), "must support");
5110 emit_int8((unsigned char)0xF3);
5111 int encode = prefix_and_encode(dst->encoding(), src->encoding());
5112 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
5113 }
5114
5115 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5116 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5117 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5118 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5119 attributes.set_embedded_opmask_register_specifier(mask);
5120 attributes.set_is_evex_instruction();
5121 if (merge) {
5122 attributes.reset_is_clear_context();
5123 }
5124 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5125 emit_int16(0x54, (0xC0 | encode));
5126 }
5127
5128 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5129 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
5130 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5131 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5132 attributes.set_is_evex_instruction();
5133 attributes.set_embedded_opmask_register_specifier(mask);
5134 if (merge) {
5135 attributes.reset_is_clear_context();
5136 }
5137 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5138 emit_int16(0x54, (0xC0 | encode));
5139 }
5140
5141 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5142 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5143 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5144 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5145 attributes.set_is_evex_instruction();
5146 attributes.set_embedded_opmask_register_specifier(mask);
5147 if (merge) {
5148 attributes.reset_is_clear_context();
5149 }
5150 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5151 emit_int16(0x55, (0xC0 | encode));
5152 }
5153
5154 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
5155 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
5156 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
5157 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5158 attributes.set_is_evex_instruction();
5159 attributes.set_embedded_opmask_register_specifier(mask);
5160 if (merge) {
5161 attributes.reset_is_clear_context();
5162 }
5163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5164 emit_int16(0x55, (0xC0 | encode));
5165 }
5166
5167 void Assembler::popf() {
5168 emit_int8((unsigned char)0x9D);
5169 }
5170
5171 #ifndef _LP64 // no 32bit push/pop on amd64
5172 void Assembler::popl(Address dst) {
5173 // NOTE: this will adjust stack by 8byte on 64bits
5174 InstructionMark im(this);
5175 prefix(dst);
5176 emit_int8((unsigned char)0x8F);
5177 emit_operand(rax, dst, 0);
5178 }
5179 #endif
5180
5181 void Assembler::prefetchnta(Address src) {
5182 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5183 InstructionMark im(this);
5184 prefix(src);
5185 emit_int16(0x0F, 0x18);
5186 emit_operand(rax, src, 0); // 0, src
5187 }
5188
5189 void Assembler::prefetchr(Address src) {
5190 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5191 InstructionMark im(this);
5192 prefix(src);
5193 emit_int16(0x0F, 0x0D);
5194 emit_operand(rax, src, 0); // 0, src
5195 }
5196
5197 void Assembler::prefetcht0(Address src) {
5198 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5199 InstructionMark im(this);
5200 prefix(src);
5201 emit_int16(0x0F, 0x18);
5202 emit_operand(rcx, src, 0); // 1, src
5203 }
5204
5205 void Assembler::prefetcht1(Address src) {
5206 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5207 InstructionMark im(this);
5208 prefix(src);
5209 emit_int16(0x0F, 0x18);
5210 emit_operand(rdx, src, 0); // 2, src
5211 }
5212
5213 void Assembler::prefetcht2(Address src) {
5214 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5215 InstructionMark im(this);
5216 prefix(src);
5217 emit_int16(0x0F, 0x18);
5218 emit_operand(rbx, src, 0); // 3, src
5219 }
5220
5221 void Assembler::prefetchw(Address src) {
5222 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5223 InstructionMark im(this);
5224 prefix(src);
5225 emit_int16(0x0F, 0x0D);
5226 emit_operand(rcx, src, 0); // 1, src
5227 }
5228
5229 void Assembler::prefix(Prefix p) {
5230 emit_int8(p);
5231 }
5232
5233 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
5234 assert(VM_Version::supports_ssse3(), "");
5235 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5236 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5237 emit_int16(0x00, (0xC0 | encode));
5238 }
5239
5240 void Assembler::evpshufb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
5241 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
5242 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5243 attributes.set_is_evex_instruction();
5244 attributes.set_embedded_opmask_register_specifier(mask);
5245 if (merge) {
5246 attributes.reset_is_clear_context();
5247 }
5248 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5249 emit_int16(0x00, (0xC0 | encode));
5250 }
5251
5252 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5253 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5254 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5255 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5256 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5257 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5258 emit_int16(0x00, (0xC0 | encode));
5259 }
5260
5261 void Assembler::pshufb(XMMRegister dst, Address src) {
5262 assert(VM_Version::supports_ssse3(), "");
5263 InstructionMark im(this);
5264 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5265 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5266 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5267 emit_int8(0x00);
5268 emit_operand(dst, src, 0);
5269 }
5270
5271 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
5272 assert(isByte(mode), "invalid value");
5273 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5274 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
5275 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5276 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5277 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5278 }
5279
5280 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5281 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5282 (vector_len == AVX_256bit? VM_Version::supports_avx2() :
5283 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
5284 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5285 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5286 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5287 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5288 }
5289
5290 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
5291 assert(isByte(mode), "invalid value");
5292 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5293 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5294 InstructionMark im(this);
5295 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5296 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5297 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5298 emit_int8(0x70);
5299 emit_operand(dst, src, 1);
5300 emit_int8(mode & 0xFF);
5301 }
5302
5303 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
5304 assert(isByte(mode), "invalid value");
5305 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5306 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5307 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5308 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5309 }
5310
5311 void Assembler::vpshufhw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5312 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5313 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5314 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
5315 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5316 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5317 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5318 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5319 }
5320
5321 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5322 assert(isByte(mode), "invalid value");
5323 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5324 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5325 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5326 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5327 }
5328
5329 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
5330 assert(isByte(mode), "invalid value");
5331 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5332 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5333 InstructionMark im(this);
5334 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5335 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5336 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5337 emit_int8(0x70);
5338 emit_operand(dst, src, 1);
5339 emit_int8(mode & 0xFF);
5340 }
5341
5342 void Assembler::vpshuflw(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5343 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5344 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5345 (vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false)), "");
5346 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5347 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5348 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5349 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5350 }
5351
5352 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5353 assert(VM_Version::supports_evex(), "requires EVEX support");
5354 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
5355 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5356 attributes.set_is_evex_instruction();
5357 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5358 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
5359 }
5360
5361 void Assembler::shufpd(XMMRegister dst, XMMRegister src, int imm8) {
5362 assert(isByte(imm8), "invalid value");
5363 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5364 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5366 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5367 }
5368
5369 void Assembler::vshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5370 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5371 attributes.set_rex_vex_w_reverted();
5372 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5373 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5374 }
5375
5376 void Assembler::shufps(XMMRegister dst, XMMRegister src, int imm8) {
5377 assert(isByte(imm8), "invalid value");
5378 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5379 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5380 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5381 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5382 }
5383
5384 void Assembler::vshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5385 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5386 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5387 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5388 }
5389
5390 void Assembler::psrldq(XMMRegister dst, int shift) {
5391 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5392 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5393 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5394 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5395 emit_int24(0x73, (0xC0 | encode), shift);
5396 }
5397
5398 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5399 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5400 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5401 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5402 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5403 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5404 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5405 }
5406
5407 void Assembler::pslldq(XMMRegister dst, int shift) {
5408 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5409 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5410 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5411 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
5412 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5413 emit_int24(0x73, (0xC0 | encode), shift);
5414 }
5415
5416 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5417 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5418 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5419 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5420 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5421 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5422 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5423 }
5424
5425 void Assembler::ptest(XMMRegister dst, Address src) {
5426 assert(VM_Version::supports_sse4_1(), "");
5427 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5428 InstructionMark im(this);
5429 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5430 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5431 emit_int8(0x17);
5432 emit_operand(dst, src, 0);
5433 }
5434
5435 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
5436 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
5437 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5438 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5439 emit_int8(0x17);
5440 emit_int8((0xC0 | encode));
5441 }
5442
5443 void Assembler::vptest(XMMRegister dst, Address src) {
5444 assert(VM_Version::supports_avx(), "");
5445 InstructionMark im(this);
5446 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5447 assert(dst != xnoreg, "sanity");
5448 // swap src<->dst for encoding
5449 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5450 emit_int8(0x17);
5451 emit_operand(dst, src, 0);
5452 }
5453
5454 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
5455 assert(VM_Version::supports_avx(), "");
5456 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5457 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5458 emit_int16(0x17, (0xC0 | encode));
5459 }
5460
5461 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
5462 assert(VM_Version::supports_avx(), "");
5463 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5464 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5465 emit_int16(0x17, (0xC0 | encode));
5466 }
5467
5468 void Assembler::vtestps(XMMRegister dst, XMMRegister src, int vector_len) {
5469 assert(VM_Version::supports_avx(), "");
5470 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5471 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5472 emit_int16(0x0E, (0xC0 | encode));
5473 }
5474
5475 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5476 assert(vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : VM_Version::supports_avx512vlbw(), "");
5477 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
5478 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5479 attributes.set_is_evex_instruction();
5480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5481 emit_int16(0x26, (0xC0 | encode));
5482 }
5483
5484 void Assembler::evptestmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5485 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
5486 // Encoding: EVEX.NDS.XXX.66.0F38.W0 DB /r
5487 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5488 attributes.set_is_evex_instruction();
5489 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5490 emit_int16(0x27, (0xC0 | encode));
5491 }
5492
5493 void Assembler::evptestnmd(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5494 assert(vector_len == AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx512vl(), "");
5495 // Encoding: EVEX.NDS.XXX.F3.0F38.W0 DB /r
5496 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5497 attributes.set_is_evex_instruction();
5498 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
5499 emit_int16(0x27, (0xC0 | encode));
5500 }
5501
5502 void Assembler::punpcklbw(XMMRegister dst, Address src) {
5503 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5504 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5505 InstructionMark im(this);
5506 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5507 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5508 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5509 emit_int8(0x60);
5510 emit_operand(dst, src, 0);
5511 }
5512
5513 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
5514 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5517 emit_int16(0x60, (0xC0 | encode));
5518 }
5519
5520 void Assembler::punpckldq(XMMRegister dst, Address src) {
5521 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5522 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5523 InstructionMark im(this);
5524 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5525 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5526 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5527 emit_int8(0x62);
5528 emit_operand(dst, src, 0);
5529 }
5530
5531 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
5532 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5533 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5534 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5535 emit_int16(0x62, (0xC0 | encode));
5536 }
5537
5538 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
5539 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5540 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5541 attributes.set_rex_vex_w_reverted();
5542 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5543 emit_int16(0x6C, (0xC0 | encode));
5544 }
5545
5546 void Assembler::evpunpcklqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5547 evpunpcklqdq(dst, k0, src1, src2, false, vector_len);
5548 }
5549
5550 void Assembler::evpunpcklqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5551 assert(VM_Version::supports_evex(), "requires AVX512F");
5552 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
5553 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5554 attributes.set_is_evex_instruction();
5555 attributes.set_embedded_opmask_register_specifier(mask);
5556 if (merge) {
5557 attributes.reset_is_clear_context();
5558 }
5559
5560 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5561 emit_int16(0x6C, (0xC0 | encode));
5562 }
5563
5564 void Assembler::evpunpckhqdq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5565 evpunpckhqdq(dst, k0, src1, src2, false, vector_len);
5566 }
5567
5568 void Assembler::evpunpckhqdq(XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
5569 assert(VM_Version::supports_evex(), "requires AVX512F");
5570 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
5571 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5572 attributes.set_is_evex_instruction();
5573 attributes.set_embedded_opmask_register_specifier(mask);
5574 if (merge) {
5575 attributes.reset_is_clear_context();
5576 }
5577
5578 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5579 emit_int16(0x6D, (0xC0 | encode));
5580 }
5581
5582 void Assembler::push(int32_t imm32) {
5583 // in 64bits we push 64bits onto the stack but only
5584 // take a 32bit immediate
5585 emit_int8(0x68);
5586 emit_int32(imm32);
5587 }
5588
5589 void Assembler::push(Register src) {
5590 int encode = prefix_and_encode(src->encoding());
5591 emit_int8(0x50 | encode);
5592 }
5593
5594 void Assembler::pushf() {
5595 emit_int8((unsigned char)0x9C);
5596 }
5597
5598 #ifndef _LP64 // no 32bit push/pop on amd64
5599 void Assembler::pushl(Address src) {
5600 // Note this will push 64bit on 64bit
5601 InstructionMark im(this);
5602 prefix(src);
5603 emit_int8((unsigned char)0xFF);
5604 emit_operand(rsi, src, 0);
5605 }
5606 #endif
5607
5608 void Assembler::rcll(Register dst, int imm8) {
5609 assert(isShiftCount(imm8), "illegal shift count");
5610 int encode = prefix_and_encode(dst->encoding());
5611 if (imm8 == 1) {
5612 emit_int16((unsigned char)0xD1, (0xD0 | encode));
5613 } else {
5614 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
5615 }
5616 }
5617
5618 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
5619 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5620 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5621 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5622 emit_int16(0x53, (0xC0 | encode));
5623 }
5624
5625 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
5626 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5627 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5628 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5629 emit_int16(0x53, (0xC0 | encode));
5630 }
5631
5632 void Assembler::rdtsc() {
5633 emit_int16(0x0F, 0x31);
5634 }
5635
5636 void Assembler::rdtscp() {
5637 emit_int24(0x0F, 0x01, (unsigned char)0xF9);
5638 }
5639
5640 // copies data from [esi] to [edi] using rcx pointer sized words
5641 // generic
5642 void Assembler::rep_mov() {
5643 // REP
5644 // MOVSQ
5645 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
5646 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xA5);)
5647 }
5648
5649 // sets rcx bytes with rax, value at [edi]
5650 void Assembler::rep_stosb() {
5651 // REP
5652 // STOSB
5653 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
5654 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAA);)
5655 }
5656
5657 // sets rcx pointer sized words with rax, value at [edi]
5658 // generic
5659 void Assembler::rep_stos() {
5660 // REP
5661 // LP64:STOSQ, LP32:STOSD
5662 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
5663 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAB);)
5664 }
5665
5666 // scans rcx pointer sized words at [edi] for occurrence of rax,
5667 // generic
5668 void Assembler::repne_scan() { // repne_scan
5669 // SCASQ
5670 LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
5671 NOT_LP64( emit_int16((unsigned char)0xF2, (unsigned char)0xAF);)
5672 }
5673
5674 #ifdef _LP64
5675 // scans rcx 4 byte words at [edi] for occurrence of rax,
5676 // generic
5677 void Assembler::repne_scanl() { // repne_scan
5678 // SCASL
5679 emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
5680 }
5681 #endif
5682
5683 void Assembler::ret(int imm16) {
5684 if (imm16 == 0) {
5685 emit_int8((unsigned char)0xC3);
5686 } else {
5687 emit_int8((unsigned char)0xC2);
5688 emit_int16(imm16);
5689 }
5690 }
5691
5692 void Assembler::roll(Register dst, int imm8) {
5693 assert(isShiftCount(imm8), "illegal shift count");
5694 int encode = prefix_and_encode(dst->encoding());
5695 if (imm8 == 1) {
5696 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5697 } else {
5698 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5699 }
5700 }
5701
5702 void Assembler::roll(Register dst) {
5703 int encode = prefix_and_encode(dst->encoding());
5704 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5705 }
5706
5707 void Assembler::rorl(Register dst, int imm8) {
5708 assert(isShiftCount(imm8), "illegal shift count");
5709 int encode = prefix_and_encode(dst->encoding());
5710 if (imm8 == 1) {
5711 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5712 } else {
5713 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5714 }
5715 }
5716
5717 void Assembler::rorl(Register dst) {
5718 int encode = prefix_and_encode(dst->encoding());
5719 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5720 }
5721
5722 #ifdef _LP64
5723 void Assembler::rorq(Register dst) {
5724 int encode = prefixq_and_encode(dst->encoding());
5725 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5726 }
5727
5728 void Assembler::rorq(Register dst, int imm8) {
5729 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5730 int encode = prefixq_and_encode(dst->encoding());
5731 if (imm8 == 1) {
5732 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5733 } else {
5734 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5735 }
5736 }
5737
5738 void Assembler::rolq(Register dst) {
5739 int encode = prefixq_and_encode(dst->encoding());
5740 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5741 }
5742
5743 void Assembler::rolq(Register dst, int imm8) {
5744 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5745 int encode = prefixq_and_encode(dst->encoding());
5746 if (imm8 == 1) {
5747 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5748 } else {
5749 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5750 }
5751 }
5752 #endif
5753
5754 void Assembler::sahf() {
5755 #ifdef _LP64
5756 // Not supported in 64bit mode
5757 ShouldNotReachHere();
5758 #endif
5759 emit_int8((unsigned char)0x9E);
5760 }
5761
5762 void Assembler::sall(Address dst, int imm8) {
5763 InstructionMark im(this);
5764 assert(isShiftCount(imm8), "illegal shift count");
5765 prefix(dst);
5766 if (imm8 == 1) {
5767 emit_int8((unsigned char)0xD1);
5768 emit_operand(as_Register(4), dst, 0);
5769 }
5770 else {
5771 emit_int8((unsigned char)0xC1);
5772 emit_operand(as_Register(4), dst, 1);
5773 emit_int8(imm8);
5774 }
5775 }
5776
5777 void Assembler::sall(Address dst) {
5778 InstructionMark im(this);
5779 prefix(dst);
5780 emit_int8((unsigned char)0xD3);
5781 emit_operand(as_Register(4), dst, 0);
5782 }
5783
5784 void Assembler::sall(Register dst, int imm8) {
5785 assert(isShiftCount(imm8), "illegal shift count");
5786 int encode = prefix_and_encode(dst->encoding());
5787 if (imm8 == 1) {
5788 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5789 } else {
5790 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5791 }
5792 }
5793
5794 void Assembler::sall(Register dst) {
5795 int encode = prefix_and_encode(dst->encoding());
5796 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5797 }
5798
5799 void Assembler::sarl(Address dst, int imm8) {
5800 assert(isShiftCount(imm8), "illegal shift count");
5801 InstructionMark im(this);
5802 prefix(dst);
5803 if (imm8 == 1) {
5804 emit_int8((unsigned char)0xD1);
5805 emit_operand(as_Register(7), dst, 0);
5806 }
5807 else {
5808 emit_int8((unsigned char)0xC1);
5809 emit_operand(as_Register(7), dst, 1);
5810 emit_int8(imm8);
5811 }
5812 }
5813
5814 void Assembler::sarl(Address dst) {
5815 InstructionMark im(this);
5816 prefix(dst);
5817 emit_int8((unsigned char)0xD3);
5818 emit_operand(as_Register(7), dst, 0);
5819 }
5820
5821 void Assembler::sarl(Register dst, int imm8) {
5822 int encode = prefix_and_encode(dst->encoding());
5823 assert(isShiftCount(imm8), "illegal shift count");
5824 if (imm8 == 1) {
5825 emit_int16((unsigned char)0xD1, (0xF8 | encode));
5826 } else {
5827 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
5828 }
5829 }
5830
5831 void Assembler::sarl(Register dst) {
5832 int encode = prefix_and_encode(dst->encoding());
5833 emit_int16((unsigned char)0xD3, (0xF8 | encode));
5834 }
5835
5836 void Assembler::sbbl(Address dst, int32_t imm32) {
5837 InstructionMark im(this);
5838 prefix(dst);
5839 emit_arith_operand(0x81, rbx, dst, imm32);
5840 }
5841
5842 void Assembler::sbbl(Register dst, int32_t imm32) {
5843 prefix(dst);
5844 emit_arith(0x81, 0xD8, dst, imm32);
5845 }
5846
5847
5848 void Assembler::sbbl(Register dst, Address src) {
5849 InstructionMark im(this);
5850 prefix(src, dst);
5851 emit_int8(0x1B);
5852 emit_operand(dst, src, 0);
5853 }
5854
5855 void Assembler::sbbl(Register dst, Register src) {
5856 (void) prefix_and_encode(dst->encoding(), src->encoding());
5857 emit_arith(0x1B, 0xC0, dst, src);
5858 }
5859
5860 void Assembler::setb(Condition cc, Register dst) {
5861 assert(0 <= cc && cc < 16, "illegal cc");
5862 int encode = prefix_and_encode(dst->encoding(), true);
5863 emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
5864 }
5865
5866 void Assembler::sete(Register dst) {
5867 int encode = prefix_and_encode(dst->encoding(), true);
5868 emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
5869 }
5870
5871 void Assembler::setl(Register dst) {
5872 int encode = prefix_and_encode(dst->encoding(), true);
5873 emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
5874 }
5875
5876 void Assembler::setne(Register dst) {
5877 int encode = prefix_and_encode(dst->encoding(), true);
5878 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
5879 }
5880
5881 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
5882 assert(VM_Version::supports_ssse3(), "");
5883 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5884 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5885 emit_int24(0x0F, (0xC0 | encode), imm8);
5886 }
5887
5888 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5889 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5890 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5891 0, "");
5892 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5893 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5894 emit_int24(0x0F, (0xC0 | encode), imm8);
5895 }
5896
5897 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5898 assert(VM_Version::supports_evex(), "");
5899 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5900 attributes.set_is_evex_instruction();
5901 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5902 emit_int24(0x3, (0xC0 | encode), imm8);
5903 }
5904
5905 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
5906 assert(VM_Version::supports_sse4_1(), "");
5907 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5908 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5909 emit_int24(0x0E, (0xC0 | encode), imm8);
5910 }
5911
5912 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
5913 assert(VM_Version::supports_sha(), "");
5914 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
5915 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
5916 }
5917
5918 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
5919 assert(VM_Version::supports_sha(), "");
5920 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5921 emit_int16((unsigned char)0xC8, (0xC0 | encode));
5922 }
5923
5924 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
5925 assert(VM_Version::supports_sha(), "");
5926 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5927 emit_int16((unsigned char)0xC9, (0xC0 | encode));
5928 }
5929
5930 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
5931 assert(VM_Version::supports_sha(), "");
5932 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5933 emit_int16((unsigned char)0xCA, (0xC0 | encode));
5934 }
5935
5936 // xmm0 is implicit additional source to this instruction.
5937 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
5938 assert(VM_Version::supports_sha(), "");
5939 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5940 emit_int16((unsigned char)0xCB, (0xC0 | encode));
5941 }
5942
5943 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
5944 assert(VM_Version::supports_sha(), "");
5945 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5946 emit_int16((unsigned char)0xCC, (0xC0 | encode));
5947 }
5948
5949 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
5950 assert(VM_Version::supports_sha(), "");
5951 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5952 emit_int16((unsigned char)0xCD, (0xC0 | encode));
5953 }
5954
5955
5956 void Assembler::shll(Register dst, int imm8) {
5957 assert(isShiftCount(imm8), "illegal shift count");
5958 int encode = prefix_and_encode(dst->encoding());
5959 if (imm8 == 1 ) {
5960 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5961 } else {
5962 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5963 }
5964 }
5965
5966 void Assembler::shll(Register dst) {
5967 int encode = prefix_and_encode(dst->encoding());
5968 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5969 }
5970
5971 void Assembler::shrl(Register dst, int imm8) {
5972 assert(isShiftCount(imm8), "illegal shift count");
5973 int encode = prefix_and_encode(dst->encoding());
5974 if (imm8 == 1) {
5975 emit_int16((unsigned char)0xD1, (0xE8 | encode));
5976 }
5977 else {
5978 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
5979 }
5980 }
5981
5982 void Assembler::shrl(Register dst) {
5983 int encode = prefix_and_encode(dst->encoding());
5984 emit_int16((unsigned char)0xD3, (0xE8 | encode));
5985 }
5986
5987 void Assembler::shrl(Address dst) {
5988 InstructionMark im(this);
5989 prefix(dst);
5990 emit_int8((unsigned char)0xD3);
5991 emit_operand(as_Register(5), dst, 0);
5992 }
5993
5994 void Assembler::shrl(Address dst, int imm8) {
5995 InstructionMark im(this);
5996 assert(isShiftCount(imm8), "illegal shift count");
5997 prefix(dst);
5998 if (imm8 == 1) {
5999 emit_int8((unsigned char)0xD1);
6000 emit_operand(as_Register(5), dst, 0);
6001 }
6002 else {
6003 emit_int8((unsigned char)0xC1);
6004 emit_operand(as_Register(5), dst, 1);
6005 emit_int8(imm8);
6006 }
6007 }
6008
6009
6010 void Assembler::shldl(Register dst, Register src) {
6011 int encode = prefix_and_encode(src->encoding(), dst->encoding());
6012 emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
6013 }
6014
6015 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6016 int encode = prefix_and_encode(src->encoding(), dst->encoding());
6017 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
6018 }
6019
6020 void Assembler::shrdl(Register dst, Register src) {
6021 int encode = prefix_and_encode(src->encoding(), dst->encoding());
6022 emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
6023 }
6024
6025 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
6026 int encode = prefix_and_encode(src->encoding(), dst->encoding());
6027 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
6028 }
6029
6030 #ifdef _LP64
6031 void Assembler::shldq(Register dst, Register src, int8_t imm8) {
6032 int encode = prefixq_and_encode(src->encoding(), dst->encoding());
6033 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
6034 }
6035
6036 void Assembler::shrdq(Register dst, Register src, int8_t imm8) {
6037 int encode = prefixq_and_encode(src->encoding(), dst->encoding());
6038 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
6039 }
6040 #endif
6041
6042 // copies a single word from [esi] to [edi]
6043 void Assembler::smovl() {
6044 emit_int8((unsigned char)0xA5);
6045 }
6046
6047 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
6048 assert(VM_Version::supports_sse4_1(), "");
6049 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6050 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6051 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
6052 }
6053
6054 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
6055 assert(VM_Version::supports_sse4_1(), "");
6056 InstructionMark im(this);
6057 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6058 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6059 emit_int8(0x0B);
6060 emit_operand(dst, src, 1);
6061 emit_int8((unsigned char)rmode);
6062 }
6063
6064 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
6065 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6066 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6067 attributes.set_rex_vex_w_reverted();
6068 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6069 emit_int16(0x51, (0xC0 | encode));
6070 }
6071
6072 void Assembler::sqrtsd(XMMRegister dst, Address src) {
6073 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6074 InstructionMark im(this);
6075 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6076 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6077 attributes.set_rex_vex_w_reverted();
6078 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6079 emit_int8(0x51);
6080 emit_operand(dst, src, 0);
6081 }
6082
6083 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
6084 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6085 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6086 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6087 emit_int16(0x51, (0xC0 | encode));
6088 }
6089
6090 void Assembler::std() {
6091 emit_int8((unsigned char)0xFD);
6092 }
6093
6094 void Assembler::sqrtss(XMMRegister dst, Address src) {
6095 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6096 InstructionMark im(this);
6097 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6098 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6099 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6100 emit_int8(0x51);
6101 emit_operand(dst, src, 0);
6102 }
6103
6104 void Assembler::stmxcsr( Address dst) {
6105 if (UseAVX > 0 ) {
6106 assert(VM_Version::supports_avx(), "");
6107 InstructionMark im(this);
6108 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6109 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6110 emit_int8((unsigned char)0xAE);
6111 emit_operand(as_Register(3), dst, 0);
6112 } else {
6113 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6114 InstructionMark im(this);
6115 prefix(dst);
6116 emit_int16(0x0F, (unsigned char)0xAE);
6117 emit_operand(as_Register(3), dst, 0);
6118 }
6119 }
6120
6121 void Assembler::subl(Address dst, int32_t imm32) {
6122 InstructionMark im(this);
6123 prefix(dst);
6124 emit_arith_operand(0x81, rbp, dst, imm32);
6125 }
6126
6127 void Assembler::subl(Address dst, Register src) {
6128 InstructionMark im(this);
6129 prefix(dst, src);
6130 emit_int8(0x29);
6131 emit_operand(src, dst, 0);
6132 }
6133
6134 void Assembler::subl(Register dst, int32_t imm32) {
6135 prefix(dst);
6136 emit_arith(0x81, 0xE8, dst, imm32);
6137 }
6138
6139 // Force generation of a 4 byte immediate value even if it fits into 8bit
6140 void Assembler::subl_imm32(Register dst, int32_t imm32) {
6141 prefix(dst);
6142 emit_arith_imm32(0x81, 0xE8, dst, imm32);
6143 }
6144
6145 void Assembler::subl(Register dst, Address src) {
6146 InstructionMark im(this);
6147 prefix(src, dst);
6148 emit_int8(0x2B);
6149 emit_operand(dst, src, 0);
6150 }
6151
6152 void Assembler::subl(Register dst, Register src) {
6153 (void) prefix_and_encode(dst->encoding(), src->encoding());
6154 emit_arith(0x2B, 0xC0, dst, src);
6155 }
6156
6157 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
6158 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6159 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6160 attributes.set_rex_vex_w_reverted();
6161 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6162 emit_int16(0x5C, (0xC0 | encode));
6163 }
6164
6165 void Assembler::subsd(XMMRegister dst, Address src) {
6166 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6167 InstructionMark im(this);
6168 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6169 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6170 attributes.set_rex_vex_w_reverted();
6171 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6172 emit_int8(0x5C);
6173 emit_operand(dst, src, 0);
6174 }
6175
6176 void Assembler::subss(XMMRegister dst, XMMRegister src) {
6177 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6178 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
6179 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6180 emit_int16(0x5C, (0xC0 | encode));
6181 }
6182
6183 void Assembler::subss(XMMRegister dst, Address src) {
6184 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6185 InstructionMark im(this);
6186 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6187 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6188 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6189 emit_int8(0x5C);
6190 emit_operand(dst, src, 0);
6191 }
6192
6193 void Assembler::testb(Register dst, int imm8) {
6194 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
6195 if (dst == rax) {
6196 emit_int8((unsigned char)0xA8);
6197 emit_int8(imm8);
6198 } else {
6199 (void) prefix_and_encode(dst->encoding(), true);
6200 emit_arith_b(0xF6, 0xC0, dst, imm8);
6201 }
6202 }
6203
6204 void Assembler::testb(Address dst, int imm8) {
6205 InstructionMark im(this);
6206 prefix(dst);
6207 emit_int8((unsigned char)0xF6);
6208 emit_operand(rax, dst, 1);
6209 emit_int8(imm8);
6210 }
6211
6212 void Assembler::testl(Address dst, int32_t imm32) {
6213 InstructionMark im(this);
6214 prefix(dst);
6215 emit_int8((unsigned char)0xF7);
6216 emit_operand(as_Register(0), dst, 4);
6217 emit_int32(imm32);
6218 }
6219
6220 void Assembler::testl(Register dst, int32_t imm32) {
6221 // not using emit_arith because test
6222 // doesn't support sign-extension of
6223 // 8bit operands
6224 if (dst == rax) {
6225 emit_int8((unsigned char)0xA9);
6226 emit_int32(imm32);
6227 } else {
6228 int encode = dst->encoding();
6229 encode = prefix_and_encode(encode);
6230 emit_int16((unsigned char)0xF7, (0xC0 | encode));
6231 emit_int32(imm32);
6232 }
6233 }
6234
6235 void Assembler::testl(Register dst, Register src) {
6236 (void) prefix_and_encode(dst->encoding(), src->encoding());
6237 emit_arith(0x85, 0xC0, dst, src);
6238 }
6239
6240 void Assembler::testl(Register dst, Address src) {
6241 InstructionMark im(this);
6242 prefix(src, dst);
6243 emit_int8((unsigned char)0x85);
6244 emit_operand(dst, src, 0);
6245 }
6246
6247 void Assembler::tzcntl(Register dst, Register src) {
6248 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6249 emit_int8((unsigned char)0xF3);
6250 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6251 emit_int24(0x0F,
6252 (unsigned char)0xBC,
6253 0xC0 | encode);
6254 }
6255
6256 void Assembler::tzcntl(Register dst, Address src) {
6257 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6258 InstructionMark im(this);
6259 emit_int8((unsigned char)0xF3);
6260 prefix(src, dst);
6261 emit_int16(0x0F, (unsigned char)0xBC);
6262 emit_operand(dst, src, 0);
6263 }
6264
6265 void Assembler::tzcntq(Register dst, Register src) {
6266 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6267 emit_int8((unsigned char)0xF3);
6268 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6269 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
6270 }
6271
6272 void Assembler::tzcntq(Register dst, Address src) {
6273 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
6274 InstructionMark im(this);
6275 emit_int8((unsigned char)0xF3);
6276 prefixq(src, dst);
6277 emit_int16(0x0F, (unsigned char)0xBC);
6278 emit_operand(dst, src, 0);
6279 }
6280
6281 void Assembler::ucomisd(XMMRegister dst, Address src) {
6282 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6283 InstructionMark im(this);
6284 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6285 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6286 attributes.set_rex_vex_w_reverted();
6287 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6288 emit_int8(0x2E);
6289 emit_operand(dst, src, 0);
6290 }
6291
6292 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
6293 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6294 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6295 attributes.set_rex_vex_w_reverted();
6296 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6297 emit_int16(0x2E, (0xC0 | encode));
6298 }
6299
6300 void Assembler::ucomiss(XMMRegister dst, Address src) {
6301 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6302 InstructionMark im(this);
6303 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6304 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6305 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6306 emit_int8(0x2E);
6307 emit_operand(dst, src, 0);
6308 }
6309
6310 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
6311 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6312 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6313 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6314 emit_int16(0x2E, (0xC0 | encode));
6315 }
6316
6317 void Assembler::xabort(int8_t imm8) {
6318 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
6319 }
6320
6321 void Assembler::xaddb(Address dst, Register src) {
6322 InstructionMark im(this);
6323 prefix(dst, src, true);
6324 emit_int16(0x0F, (unsigned char)0xC0);
6325 emit_operand(src, dst, 0);
6326 }
6327
6328 void Assembler::xaddw(Address dst, Register src) {
6329 InstructionMark im(this);
6330 emit_int8(0x66);
6331 prefix(dst, src);
6332 emit_int16(0x0F, (unsigned char)0xC1);
6333 emit_operand(src, dst, 0);
6334 }
6335
6336 void Assembler::xaddl(Address dst, Register src) {
6337 InstructionMark im(this);
6338 prefix(dst, src);
6339 emit_int16(0x0F, (unsigned char)0xC1);
6340 emit_operand(src, dst, 0);
6341 }
6342
6343 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
6344 InstructionMark im(this);
6345 relocate(rtype);
6346 if (abort.is_bound()) {
6347 address entry = target(abort);
6348 assert(entry != NULL, "abort entry NULL");
6349 intptr_t offset = entry - pc();
6350 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6351 emit_int32(offset - 6); // 2 opcode + 4 address
6352 } else {
6353 abort.add_patch_at(code(), locator());
6354 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6355 emit_int32(0);
6356 }
6357 }
6358
6359 void Assembler::xchgb(Register dst, Address src) { // xchg
6360 InstructionMark im(this);
6361 prefix(src, dst, true);
6362 emit_int8((unsigned char)0x86);
6363 emit_operand(dst, src, 0);
6364 }
6365
6366 void Assembler::xchgw(Register dst, Address src) { // xchg
6367 InstructionMark im(this);
6368 emit_int8(0x66);
6369 prefix(src, dst);
6370 emit_int8((unsigned char)0x87);
6371 emit_operand(dst, src, 0);
6372 }
6373
6374 void Assembler::xchgl(Register dst, Address src) { // xchg
6375 InstructionMark im(this);
6376 prefix(src, dst);
6377 emit_int8((unsigned char)0x87);
6378 emit_operand(dst, src, 0);
6379 }
6380
6381 void Assembler::xchgl(Register dst, Register src) {
6382 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6383 emit_int16((unsigned char)0x87, (0xC0 | encode));
6384 }
6385
6386 void Assembler::xend() {
6387 emit_int24(0x0F, 0x01, (unsigned char)0xD5);
6388 }
6389
6390 void Assembler::xgetbv() {
6391 emit_int24(0x0F, 0x01, (unsigned char)0xD0);
6392 }
6393
6394 void Assembler::xorl(Address dst, int32_t imm32) {
6395 InstructionMark im(this);
6396 prefix(dst);
6397 emit_arith_operand(0x81, as_Register(6), dst, imm32);
6398 }
6399
6400 void Assembler::xorl(Register dst, int32_t imm32) {
6401 prefix(dst);
6402 emit_arith(0x81, 0xF0, dst, imm32);
6403 }
6404
6405 void Assembler::xorl(Register dst, Address src) {
6406 InstructionMark im(this);
6407 prefix(src, dst);
6408 emit_int8(0x33);
6409 emit_operand(dst, src, 0);
6410 }
6411
6412 void Assembler::xorl(Register dst, Register src) {
6413 (void) prefix_and_encode(dst->encoding(), src->encoding());
6414 emit_arith(0x33, 0xC0, dst, src);
6415 }
6416
6417 void Assembler::xorl(Address dst, Register src) {
6418 InstructionMark im(this);
6419 prefix(dst, src);
6420 emit_int8(0x31);
6421 emit_operand(src, dst, 0);
6422 }
6423
6424 void Assembler::xorb(Register dst, Address src) {
6425 InstructionMark im(this);
6426 prefix(src, dst);
6427 emit_int8(0x32);
6428 emit_operand(dst, src, 0);
6429 }
6430
6431 void Assembler::xorb(Address dst, Register src) {
6432 InstructionMark im(this);
6433 prefix(dst, src, true);
6434 emit_int8(0x30);
6435 emit_operand(src, dst, 0);
6436 }
6437
6438 void Assembler::xorw(Register dst, Register src) {
6439 (void)prefix_and_encode(dst->encoding(), src->encoding());
6440 emit_arith(0x33, 0xC0, dst, src);
6441 }
6442
6443 // AVX 3-operands scalar float-point arithmetic instructions
6444
6445 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
6446 assert(VM_Version::supports_avx(), "");
6447 InstructionMark im(this);
6448 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6449 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6450 attributes.set_rex_vex_w_reverted();
6451 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6452 emit_int8(0x58);
6453 emit_operand(dst, src, 0);
6454 }
6455
6456 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6457 assert(VM_Version::supports_avx(), "");
6458 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6459 attributes.set_rex_vex_w_reverted();
6460 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6461 emit_int16(0x58, (0xC0 | encode));
6462 }
6463
6464 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
6465 assert(VM_Version::supports_avx(), "");
6466 InstructionMark im(this);
6467 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6468 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6469 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6470 emit_int8(0x58);
6471 emit_operand(dst, src, 0);
6472 }
6473
6474 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6475 assert(VM_Version::supports_avx(), "");
6476 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6477 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6478 emit_int16(0x58, (0xC0 | encode));
6479 }
6480
6481 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
6482 assert(VM_Version::supports_avx(), "");
6483 InstructionMark im(this);
6484 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6485 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6486 attributes.set_rex_vex_w_reverted();
6487 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6488 emit_int8(0x5E);
6489 emit_operand(dst, src, 0);
6490 }
6491
6492 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6493 assert(VM_Version::supports_avx(), "");
6494 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6495 attributes.set_rex_vex_w_reverted();
6496 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6497 emit_int16(0x5E, (0xC0 | encode));
6498 }
6499
6500 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
6501 assert(VM_Version::supports_avx(), "");
6502 InstructionMark im(this);
6503 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6504 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6505 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6506 emit_int8(0x5E);
6507 emit_operand(dst, src, 0);
6508 }
6509
6510 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6511 assert(VM_Version::supports_avx(), "");
6512 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6513 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6514 emit_int16(0x5E, (0xC0 | encode));
6515 }
6516
6517 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6518 assert(VM_Version::supports_fma(), "");
6519 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6520 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6521 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6522 }
6523
6524 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6525 assert(VM_Version::supports_fma(), "");
6526 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6527 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6528 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6529 }
6530
6531 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
6532 assert(VM_Version::supports_avx(), "");
6533 InstructionMark im(this);
6534 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6535 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6536 attributes.set_rex_vex_w_reverted();
6537 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6538 emit_int8(0x59);
6539 emit_operand(dst, src, 0);
6540 }
6541
6542 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6543 assert(VM_Version::supports_avx(), "");
6544 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6545 attributes.set_rex_vex_w_reverted();
6546 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6547 emit_int16(0x59, (0xC0 | encode));
6548 }
6549
6550 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
6551 assert(VM_Version::supports_avx(), "");
6552 InstructionMark im(this);
6553 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6554 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6555 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6556 emit_int8(0x59);
6557 emit_operand(dst, src, 0);
6558 }
6559
6560 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6561 assert(VM_Version::supports_avx(), "");
6562 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6563 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6564 emit_int16(0x59, (0xC0 | encode));
6565 }
6566
6567 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
6568 assert(VM_Version::supports_avx(), "");
6569 InstructionMark im(this);
6570 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6571 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6572 attributes.set_rex_vex_w_reverted();
6573 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6574 emit_int8(0x5C);
6575 emit_operand(dst, src, 0);
6576 }
6577
6578 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6579 assert(VM_Version::supports_avx(), "");
6580 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6581 attributes.set_rex_vex_w_reverted();
6582 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6583 emit_int16(0x5C, (0xC0 | encode));
6584 }
6585
6586 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
6587 assert(VM_Version::supports_avx(), "");
6588 InstructionMark im(this);
6589 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6590 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6591 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6592 emit_int8(0x5C);
6593 emit_operand(dst, src, 0);
6594 }
6595
6596 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6597 assert(VM_Version::supports_avx(), "");
6598 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6600 emit_int16(0x5C, (0xC0 | encode));
6601 }
6602
6603 //====================VECTOR ARITHMETIC=====================================
6604
6605 // Float-point vector arithmetic
6606
6607 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
6608 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6609 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6610 attributes.set_rex_vex_w_reverted();
6611 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6612 emit_int16(0x58, (0xC0 | encode));
6613 }
6614
6615 void Assembler::addpd(XMMRegister dst, Address src) {
6616 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6617 InstructionMark im(this);
6618 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6619 attributes.set_rex_vex_w_reverted();
6620 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6621 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6622 emit_int8(0x58);
6623 emit_operand(dst, src, 0);
6624 }
6625
6626
6627 void Assembler::addps(XMMRegister dst, XMMRegister src) {
6628 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6629 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6630 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6631 emit_int16(0x58, (0xC0 | encode));
6632 }
6633
6634 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6635 assert(VM_Version::supports_avx(), "");
6636 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6637 attributes.set_rex_vex_w_reverted();
6638 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6639 emit_int16(0x58, (0xC0 | encode));
6640 }
6641
6642 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6643 assert(VM_Version::supports_avx(), "");
6644 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6645 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6646 emit_int16(0x58, (0xC0 | encode));
6647 }
6648
6649 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6650 assert(VM_Version::supports_avx(), "");
6651 InstructionMark im(this);
6652 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6653 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6654 attributes.set_rex_vex_w_reverted();
6655 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6656 emit_int8(0x58);
6657 emit_operand(dst, src, 0);
6658 }
6659
6660 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6661 assert(VM_Version::supports_avx(), "");
6662 InstructionMark im(this);
6663 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6664 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6665 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6666 emit_int8(0x58);
6667 emit_operand(dst, src, 0);
6668 }
6669
6670 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
6671 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6672 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6673 attributes.set_rex_vex_w_reverted();
6674 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6675 emit_int16(0x5C, (0xC0 | encode));
6676 }
6677
6678 void Assembler::subps(XMMRegister dst, XMMRegister src) {
6679 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6680 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6681 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6682 emit_int16(0x5C, (0xC0 | encode));
6683 }
6684
6685 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6686 assert(VM_Version::supports_avx(), "");
6687 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6688 attributes.set_rex_vex_w_reverted();
6689 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6690 emit_int16(0x5C, (0xC0 | encode));
6691 }
6692
6693 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6694 assert(VM_Version::supports_avx(), "");
6695 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6696 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6697 emit_int16(0x5C, (0xC0 | encode));
6698 }
6699
6700 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6701 assert(VM_Version::supports_avx(), "");
6702 InstructionMark im(this);
6703 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6704 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6705 attributes.set_rex_vex_w_reverted();
6706 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6707 emit_int8(0x5C);
6708 emit_operand(dst, src, 0);
6709 }
6710
6711 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6712 assert(VM_Version::supports_avx(), "");
6713 InstructionMark im(this);
6714 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6715 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6716 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6717 emit_int8(0x5C);
6718 emit_operand(dst, src, 0);
6719 }
6720
6721 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
6722 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6723 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6724 attributes.set_rex_vex_w_reverted();
6725 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6726 emit_int16(0x59, (0xC0 | encode));
6727 }
6728
6729 void Assembler::mulpd(XMMRegister dst, Address src) {
6730 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6731 InstructionMark im(this);
6732 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6733 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6734 attributes.set_rex_vex_w_reverted();
6735 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6736 emit_int8(0x59);
6737 emit_operand(dst, src, 0);
6738 }
6739
6740 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
6741 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6742 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6743 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6744 emit_int16(0x59, (0xC0 | encode));
6745 }
6746
6747 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6748 assert(VM_Version::supports_avx(), "");
6749 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6750 attributes.set_rex_vex_w_reverted();
6751 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6752 emit_int16(0x59, (0xC0 | encode));
6753 }
6754
6755 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6756 assert(VM_Version::supports_avx(), "");
6757 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6758 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6759 emit_int16(0x59, (0xC0 | encode));
6760 }
6761
6762 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6763 assert(VM_Version::supports_avx(), "");
6764 InstructionMark im(this);
6765 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6766 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6767 attributes.set_rex_vex_w_reverted();
6768 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6769 emit_int8(0x59);
6770 emit_operand(dst, src, 0);
6771 }
6772
6773 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6774 assert(VM_Version::supports_avx(), "");
6775 InstructionMark im(this);
6776 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6777 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6778 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6779 emit_int8(0x59);
6780 emit_operand(dst, src, 0);
6781 }
6782
6783 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6784 assert(VM_Version::supports_fma(), "");
6785 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6786 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6787 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6788 }
6789
6790 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6791 assert(VM_Version::supports_fma(), "");
6792 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6793 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6794 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6795 }
6796
6797 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6798 assert(VM_Version::supports_fma(), "");
6799 InstructionMark im(this);
6800 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6801 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6802 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6803 emit_int8((unsigned char)0xB8);
6804 emit_operand(dst, src2, 0);
6805 }
6806
6807 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6808 assert(VM_Version::supports_fma(), "");
6809 InstructionMark im(this);
6810 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6811 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6812 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6813 emit_int8((unsigned char)0xB8);
6814 emit_operand(dst, src2, 0);
6815 }
6816
6817 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
6818 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6819 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6820 attributes.set_rex_vex_w_reverted();
6821 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6822 emit_int16(0x5E, (0xC0 | encode));
6823 }
6824
6825 void Assembler::divps(XMMRegister dst, XMMRegister src) {
6826 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6827 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6828 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6829 emit_int16(0x5E, (0xC0 | encode));
6830 }
6831
6832 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6833 assert(VM_Version::supports_avx(), "");
6834 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6835 attributes.set_rex_vex_w_reverted();
6836 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6837 emit_int16(0x5E, (0xC0 | encode));
6838 }
6839
6840 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6841 assert(VM_Version::supports_avx(), "");
6842 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6843 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6844 emit_int16(0x5E, (0xC0 | encode));
6845 }
6846
6847 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6848 assert(VM_Version::supports_avx(), "");
6849 InstructionMark im(this);
6850 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6851 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6852 attributes.set_rex_vex_w_reverted();
6853 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6854 emit_int8(0x5E);
6855 emit_operand(dst, src, 0);
6856 }
6857
6858 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6859 assert(VM_Version::supports_avx(), "");
6860 InstructionMark im(this);
6861 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6862 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6863 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6864 emit_int8(0x5E);
6865 emit_operand(dst, src, 0);
6866 }
6867
6868 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6869 assert(VM_Version::supports_avx(), "");
6870 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6871 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6872 emit_int24(0x09, (0xC0 | encode), (rmode));
6873 }
6874
6875 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6876 assert(VM_Version::supports_avx(), "");
6877 InstructionMark im(this);
6878 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6879 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6880 emit_int8(0x09);
6881 emit_operand(dst, src, 1);
6882 emit_int8((rmode));
6883 }
6884
6885 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6886 assert(VM_Version::supports_evex(), "requires EVEX support");
6887 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6888 attributes.set_is_evex_instruction();
6889 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6890 emit_int24(0x09, (0xC0 | encode), (rmode));
6891 }
6892
6893 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6894 assert(VM_Version::supports_evex(), "requires EVEX support");
6895 assert(dst != xnoreg, "sanity");
6896 InstructionMark im(this);
6897 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6898 attributes.set_is_evex_instruction();
6899 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6900 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6901 emit_int8(0x09);
6902 emit_operand(dst, src, 1);
6903 emit_int8((rmode));
6904 }
6905
6906 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
6907 assert(VM_Version::supports_avx(), "");
6908 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6909 attributes.set_rex_vex_w_reverted();
6910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6911 emit_int16(0x51, (0xC0 | encode));
6912 }
6913
6914 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
6915 assert(VM_Version::supports_avx(), "");
6916 InstructionMark im(this);
6917 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6918 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6919 attributes.set_rex_vex_w_reverted();
6920 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6921 emit_int8(0x51);
6922 emit_operand(dst, src, 0);
6923 }
6924
6925 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
6926 assert(VM_Version::supports_avx(), "");
6927 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6928 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6929 emit_int16(0x51, (0xC0 | encode));
6930 }
6931
6932 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
6933 assert(VM_Version::supports_avx(), "");
6934 InstructionMark im(this);
6935 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6936 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6937 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6938 emit_int8(0x51);
6939 emit_operand(dst, src, 0);
6940 }
6941
6942 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
6943 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6944 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6945 attributes.set_rex_vex_w_reverted();
6946 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6947 emit_int16(0x54, (0xC0 | encode));
6948 }
6949
6950 void Assembler::andps(XMMRegister dst, XMMRegister src) {
6951 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6952 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6953 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6954 emit_int16(0x54, (0xC0 | encode));
6955 }
6956
6957 void Assembler::andps(XMMRegister dst, Address src) {
6958 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6959 InstructionMark im(this);
6960 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6961 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6962 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6963 emit_int8(0x54);
6964 emit_operand(dst, src, 0);
6965 }
6966
6967 void Assembler::andpd(XMMRegister dst, Address src) {
6968 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6969 InstructionMark im(this);
6970 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6971 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6972 attributes.set_rex_vex_w_reverted();
6973 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6974 emit_int8(0x54);
6975 emit_operand(dst, src, 0);
6976 }
6977
6978 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6979 assert(VM_Version::supports_avx(), "");
6980 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6981 attributes.set_rex_vex_w_reverted();
6982 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6983 emit_int16(0x54, (0xC0 | encode));
6984 }
6985
6986 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6987 assert(VM_Version::supports_avx(), "");
6988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6989 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6990 emit_int16(0x54, (0xC0 | encode));
6991 }
6992
6993 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6994 assert(VM_Version::supports_avx(), "");
6995 InstructionMark im(this);
6996 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6997 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6998 attributes.set_rex_vex_w_reverted();
6999 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7000 emit_int8(0x54);
7001 emit_operand(dst, src, 0);
7002 }
7003
7004 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7005 assert(VM_Version::supports_avx(), "");
7006 InstructionMark im(this);
7007 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7008 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7009 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7010 emit_int8(0x54);
7011 emit_operand(dst, src, 0);
7012 }
7013
7014 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
7015 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7016 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7017 attributes.set_rex_vex_w_reverted();
7018 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7019 emit_int8(0x15);
7020 emit_int8((0xC0 | encode));
7021 }
7022
7023 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
7024 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7025 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7026 attributes.set_rex_vex_w_reverted();
7027 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7028 emit_int16(0x14, (0xC0 | encode));
7029 }
7030
7031 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
7032 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7033 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7034 attributes.set_rex_vex_w_reverted();
7035 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7036 emit_int16(0x57, (0xC0 | encode));
7037 }
7038
7039 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
7040 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7041 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7042 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7043 emit_int16(0x57, (0xC0 | encode));
7044 }
7045
7046 void Assembler::xorpd(XMMRegister dst, Address src) {
7047 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7048 InstructionMark im(this);
7049 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7050 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7051 attributes.set_rex_vex_w_reverted();
7052 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7053 emit_int8(0x57);
7054 emit_operand(dst, src, 0);
7055 }
7056
7057 void Assembler::xorps(XMMRegister dst, Address src) {
7058 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7059 InstructionMark im(this);
7060 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7061 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7062 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7063 emit_int8(0x57);
7064 emit_operand(dst, src, 0);
7065 }
7066
7067 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7068 assert(VM_Version::supports_avx(), "");
7069 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7070 attributes.set_rex_vex_w_reverted();
7071 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7072 emit_int16(0x57, (0xC0 | encode));
7073 }
7074
7075 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7076 assert(VM_Version::supports_avx(), "");
7077 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7078 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7079 emit_int16(0x57, (0xC0 | encode));
7080 }
7081
7082 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7083 assert(VM_Version::supports_avx(), "");
7084 InstructionMark im(this);
7085 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7086 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7087 attributes.set_rex_vex_w_reverted();
7088 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7089 emit_int8(0x57);
7090 emit_operand(dst, src, 0);
7091 }
7092
7093 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7094 assert(VM_Version::supports_avx(), "");
7095 InstructionMark im(this);
7096 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7097 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7098 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7099 emit_int8(0x57);
7100 emit_operand(dst, src, 0);
7101 }
7102
7103 // Integer vector arithmetic
7104 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7105 assert(VM_Version::supports_avx() && (vector_len == 0) ||
7106 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
7107 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7108 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7109 emit_int16(0x01, (0xC0 | encode));
7110 }
7111
7112 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7113 assert(VM_Version::supports_avx() && (vector_len == 0) ||
7114 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
7115 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7116 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7117 emit_int16(0x02, (0xC0 | encode));
7118 }
7119
7120 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
7121 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7122 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7123 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7124 emit_int16((unsigned char)0xFC, (0xC0 | encode));
7125 }
7126
7127 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
7128 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7129 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7130 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7131 emit_int16((unsigned char)0xFD, (0xC0 | encode));
7132 }
7133
7134 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
7135 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7137 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7138 emit_int16((unsigned char)0xFE, (0xC0 | encode));
7139 }
7140
7141 void Assembler::paddd(XMMRegister dst, Address src) {
7142 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7143 InstructionMark im(this);
7144 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7145 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7146 emit_int8((unsigned char)0xFE);
7147 emit_operand(dst, src, 0);
7148 }
7149
7150 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
7151 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7152 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7153 attributes.set_rex_vex_w_reverted();
7154 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7155 emit_int16((unsigned char)0xD4, (0xC0 | encode));
7156 }
7157
7158 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
7159 assert(VM_Version::supports_sse3(), "");
7160 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7161 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7162 emit_int16(0x01, (0xC0 | encode));
7163 }
7164
7165 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
7166 assert(VM_Version::supports_sse3(), "");
7167 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7168 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7169 emit_int16(0x02, (0xC0 | encode));
7170 }
7171
7172 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7173 assert(UseAVX > 0, "requires some form of AVX");
7174 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7175 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7176 emit_int16((unsigned char)0xFC, (0xC0 | encode));
7177 }
7178
7179 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7180 assert(UseAVX > 0, "requires some form of AVX");
7181 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7182 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7183 emit_int16((unsigned char)0xFD, (0xC0 | encode));
7184 }
7185
7186 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7187 assert(UseAVX > 0, "requires some form of AVX");
7188 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7189 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7190 emit_int16((unsigned char)0xFE, (0xC0 | encode));
7191 }
7192
7193 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7194 assert(UseAVX > 0, "requires some form of AVX");
7195 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7196 attributes.set_rex_vex_w_reverted();
7197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7198 emit_int16((unsigned char)0xD4, (0xC0 | encode));
7199 }
7200
7201 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7202 assert(UseAVX > 0, "requires some form of AVX");
7203 InstructionMark im(this);
7204 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7205 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7206 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7207 emit_int8((unsigned char)0xFC);
7208 emit_operand(dst, src, 0);
7209 }
7210
7211 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7212 assert(UseAVX > 0, "requires some form of AVX");
7213 InstructionMark im(this);
7214 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7215 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7216 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7217 emit_int8((unsigned char)0xFD);
7218 emit_operand(dst, src, 0);
7219 }
7220
7221 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7222 assert(UseAVX > 0, "requires some form of AVX");
7223 InstructionMark im(this);
7224 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7225 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7226 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7227 emit_int8((unsigned char)0xFE);
7228 emit_operand(dst, src, 0);
7229 }
7230
7231 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7232 assert(UseAVX > 0, "requires some form of AVX");
7233 InstructionMark im(this);
7234 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7235 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7236 attributes.set_rex_vex_w_reverted();
7237 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7238 emit_int8((unsigned char)0xD4);
7239 emit_operand(dst, src, 0);
7240 }
7241
7242 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
7243 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7244 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7245 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7246 emit_int16((unsigned char)0xF8, (0xC0 | encode));
7247 }
7248
7249 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
7250 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7251 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7252 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7253 emit_int16((unsigned char)0xF9, (0xC0 | encode));
7254 }
7255
7256 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
7257 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7258 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7259 emit_int16((unsigned char)0xFA, (0xC0 | encode));
7260 }
7261
7262 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
7263 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7264 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7265 attributes.set_rex_vex_w_reverted();
7266 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7267 emit_int8((unsigned char)0xFB);
7268 emit_int8((0xC0 | encode));
7269 }
7270
7271 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7272 assert(UseAVX > 0, "requires some form of AVX");
7273 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7275 emit_int16((unsigned char)0xD8, (0xC0 | encode));
7276 }
7277
7278 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7279 assert(UseAVX > 0, "requires some form of AVX");
7280 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7282 emit_int16((unsigned char)0xF8, (0xC0 | encode));
7283 }
7284
7285 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7286 assert(UseAVX > 0, "requires some form of AVX");
7287 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7288 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7289 emit_int16((unsigned char)0xF9, (0xC0 | encode));
7290 }
7291
7292 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7293 assert(UseAVX > 0, "requires some form of AVX");
7294 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7295 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7296 emit_int16((unsigned char)0xFA, (0xC0 | encode));
7297 }
7298
7299 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7300 assert(UseAVX > 0, "requires some form of AVX");
7301 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7302 attributes.set_rex_vex_w_reverted();
7303 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7304 emit_int16((unsigned char)0xFB, (0xC0 | encode));
7305 }
7306
7307 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7308 assert(UseAVX > 0, "requires some form of AVX");
7309 InstructionMark im(this);
7310 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7311 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7312 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7313 emit_int8((unsigned char)0xF8);
7314 emit_operand(dst, src, 0);
7315 }
7316
7317 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7318 assert(UseAVX > 0, "requires some form of AVX");
7319 InstructionMark im(this);
7320 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7321 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7322 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7323 emit_int8((unsigned char)0xF9);
7324 emit_operand(dst, src, 0);
7325 }
7326
7327 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7328 assert(UseAVX > 0, "requires some form of AVX");
7329 InstructionMark im(this);
7330 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7331 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7332 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7333 emit_int8((unsigned char)0xFA);
7334 emit_operand(dst, src, 0);
7335 }
7336
7337 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7338 assert(UseAVX > 0, "requires some form of AVX");
7339 InstructionMark im(this);
7340 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7341 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7342 attributes.set_rex_vex_w_reverted();
7343 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7344 emit_int8((unsigned char)0xFB);
7345 emit_operand(dst, src, 0);
7346 }
7347
7348 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
7349 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7351 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7352 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7353 }
7354
7355 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
7356 assert(VM_Version::supports_sse4_1(), "");
7357 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7358 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7359 emit_int16(0x40, (0xC0 | encode));
7360 }
7361
7362 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
7363 assert(VM_Version::supports_sse2(), "");
7364 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7366 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7367 }
7368
7369 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7370 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
7371 (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
7372 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
7373 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7374 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7375 emit_int16((unsigned char)0xE4, (0xC0 | encode));
7376 }
7377
7378 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7379 assert(UseAVX > 0, "requires some form of AVX");
7380 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7381 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7382 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7383 }
7384
7385 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7386 assert(UseAVX > 0, "requires some form of AVX");
7387 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7388 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7389 emit_int16(0x40, (0xC0 | encode));
7390 }
7391
7392 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7393 assert(UseAVX > 2, "requires some form of EVEX");
7394 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7395 attributes.set_is_evex_instruction();
7396 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7397 emit_int16(0x40, (0xC0 | encode));
7398 }
7399
7400 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7401 assert(UseAVX > 0, "requires some form of AVX");
7402 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7403 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7404 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7405 }
7406
7407 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7408 assert(UseAVX > 0, "requires some form of AVX");
7409 InstructionMark im(this);
7410 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7411 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7412 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7413 emit_int8((unsigned char)0xD5);
7414 emit_operand(dst, src, 0);
7415 }
7416
7417 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7418 assert(UseAVX > 0, "requires some form of AVX");
7419 InstructionMark im(this);
7420 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7421 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7422 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7423 emit_int8(0x40);
7424 emit_operand(dst, src, 0);
7425 }
7426
7427 void Assembler::evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7428 assert(UseAVX > 2, "requires some form of EVEX");
7429 InstructionMark im(this);
7430 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7431 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7432 attributes.set_is_evex_instruction();
7433 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7434 emit_int8(0x40);
7435 emit_operand(dst, src, 0);
7436 }
7437
7438 // Min, max
7439 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
7440 assert(VM_Version::supports_sse4_1(), "");
7441 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7442 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7443 emit_int16(0x38, (0xC0 | encode));
7444 }
7445
7446 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7447 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7448 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7450 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7451 emit_int16(0x38, (0xC0 | encode));
7452 }
7453
7454 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
7455 assert(VM_Version::supports_sse2(), "");
7456 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7457 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7458 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7459 }
7460
7461 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7462 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7463 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7465 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7466 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7467 }
7468
7469 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
7470 assert(VM_Version::supports_sse4_1(), "");
7471 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7472 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7473 emit_int16(0x39, (0xC0 | encode));
7474 }
7475
7476 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7477 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7478 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7479 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7481 emit_int16(0x39, (0xC0 | encode));
7482 }
7483
7484 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7485 assert(UseAVX > 2, "requires AVX512F");
7486 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7487 attributes.set_is_evex_instruction();
7488 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7489 emit_int16(0x39, (0xC0 | encode));
7490 }
7491
7492 void Assembler::minps(XMMRegister dst, XMMRegister src) {
7493 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7494 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7495 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7496 emit_int16(0x5D, (0xC0 | encode));
7497 }
7498 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7499 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7500 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7501 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7502 emit_int16(0x5D, (0xC0 | encode));
7503 }
7504
7505 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
7506 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7508 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7509 emit_int16(0x5D, (0xC0 | encode));
7510 }
7511 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7512 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7513 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7514 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7515 emit_int16(0x5D, (0xC0 | encode));
7516 }
7517
7518 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
7519 assert(VM_Version::supports_sse4_1(), "");
7520 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7521 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7522 emit_int16(0x3C, (0xC0 | encode));
7523 }
7524
7525 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7526 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7527 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7528 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7530 emit_int16(0x3C, (0xC0 | encode));
7531 }
7532
7533 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
7534 assert(VM_Version::supports_sse2(), "");
7535 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7536 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7537 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7538 }
7539
7540 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7541 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7542 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7543 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7544 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7545 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7546 }
7547
7548 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
7549 assert(VM_Version::supports_sse4_1(), "");
7550 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7551 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7552 emit_int16(0x3D, (0xC0 | encode));
7553 }
7554
7555 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7556 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7557 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7558 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7559 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7560 emit_int16(0x3D, (0xC0 | encode));
7561 }
7562
7563 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7564 assert(UseAVX > 2, "requires AVX512F");
7565 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7566 attributes.set_is_evex_instruction();
7567 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7568 emit_int16(0x3D, (0xC0 | encode));
7569 }
7570
7571 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
7572 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7573 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7574 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7575 emit_int16(0x5F, (0xC0 | encode));
7576 }
7577
7578 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7579 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7580 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7581 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7582 emit_int16(0x5F, (0xC0 | encode));
7583 }
7584
7585 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
7586 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7587 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7588 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7589 emit_int16(0x5F, (0xC0 | encode));
7590 }
7591
7592 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7593 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7594 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7596 emit_int16(0x5F, (0xC0 | encode));
7597 }
7598
7599 // Shift packed integers left by specified number of bits.
7600 void Assembler::psllw(XMMRegister dst, int shift) {
7601 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7602 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7603 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7604 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7605 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7606 }
7607
7608 void Assembler::pslld(XMMRegister dst, int shift) {
7609 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7610 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7611 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7612 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7613 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7614 }
7615
7616 void Assembler::psllq(XMMRegister dst, int shift) {
7617 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7618 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7619 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7620 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7621 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7622 }
7623
7624 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
7625 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7626 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7627 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7628 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7629 }
7630
7631 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
7632 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7633 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7634 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7635 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7636 }
7637
7638 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
7639 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7640 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7641 attributes.set_rex_vex_w_reverted();
7642 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7643 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7644 }
7645
7646 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7647 assert(UseAVX > 0, "requires some form of AVX");
7648 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7649 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7650 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7651 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7652 }
7653
7654 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7655 assert(UseAVX > 0, "requires some form of AVX");
7656 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7657 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7658 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7659 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7660 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7661 }
7662
7663 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7664 assert(UseAVX > 0, "requires some form of AVX");
7665 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7666 attributes.set_rex_vex_w_reverted();
7667 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7668 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7669 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7670 }
7671
7672 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7673 assert(UseAVX > 0, "requires some form of AVX");
7674 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7675 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7676 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7677 }
7678
7679 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7680 assert(UseAVX > 0, "requires some form of AVX");
7681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7682 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7683 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7684 }
7685
7686 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7687 assert(UseAVX > 0, "requires some form of AVX");
7688 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7689 attributes.set_rex_vex_w_reverted();
7690 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7691 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7692 }
7693
7694 // Shift packed integers logically right by specified number of bits.
7695 void Assembler::psrlw(XMMRegister dst, int shift) {
7696 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7697 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7698 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7699 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7700 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7701 }
7702
7703 void Assembler::psrld(XMMRegister dst, int shift) {
7704 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7705 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7706 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7707 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7708 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7709 }
7710
7711 void Assembler::psrlq(XMMRegister dst, int shift) {
7712 // Do not confuse it with psrldq SSE2 instruction which
7713 // shifts 128 bit value in xmm register by number of bytes.
7714 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7715 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7716 attributes.set_rex_vex_w_reverted();
7717 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7718 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7719 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7720 }
7721
7722 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
7723 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7725 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7726 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7727 }
7728
7729 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
7730 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7731 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7732 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7733 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7734 }
7735
7736 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
7737 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7738 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7739 attributes.set_rex_vex_w_reverted();
7740 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7741 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7742 }
7743
7744 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7745 assert(UseAVX > 0, "requires some form of AVX");
7746 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7747 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7748 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7749 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7750 }
7751
7752 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7753 assert(UseAVX > 0, "requires some form of AVX");
7754 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7755 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7756 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7757 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7758 }
7759
7760 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7761 assert(UseAVX > 0, "requires some form of AVX");
7762 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7763 attributes.set_rex_vex_w_reverted();
7764 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7765 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7766 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7767 }
7768
7769 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7770 assert(UseAVX > 0, "requires some form of AVX");
7771 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7772 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7773 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7774 }
7775
7776 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7777 assert(UseAVX > 0, "requires some form of AVX");
7778 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7779 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7780 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7781 }
7782
7783 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7784 assert(UseAVX > 0, "requires some form of AVX");
7785 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7786 attributes.set_rex_vex_w_reverted();
7787 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7788 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7789 }
7790
7791 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7792 assert(VM_Version::supports_avx512bw(), "");
7793 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7794 attributes.set_is_evex_instruction();
7795 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7796 emit_int16(0x10, (0xC0 | encode));
7797 }
7798
7799 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7800 assert(VM_Version::supports_avx512bw(), "");
7801 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7802 attributes.set_is_evex_instruction();
7803 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7804 emit_int16(0x12, (0xC0 | encode));
7805 }
7806
7807 // Shift packed integers arithmetically right by specified number of bits.
7808 void Assembler::psraw(XMMRegister dst, int shift) {
7809 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7810 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7811 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7812 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7813 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7814 }
7815
7816 void Assembler::psrad(XMMRegister dst, int shift) {
7817 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7818 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7819 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
7820 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7821 emit_int8(0x72);
7822 emit_int8((0xC0 | encode));
7823 emit_int8(shift & 0xFF);
7824 }
7825
7826 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
7827 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7828 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7829 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7830 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7831 }
7832
7833 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
7834 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7835 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7836 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7837 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7838 }
7839
7840 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7841 assert(UseAVX > 0, "requires some form of AVX");
7842 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7843 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7844 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7845 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7846 }
7847
7848 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7849 assert(UseAVX > 0, "requires some form of AVX");
7850 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7851 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7852 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7853 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7854 }
7855
7856 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7857 assert(UseAVX > 0, "requires some form of AVX");
7858 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7859 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7860 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7861 }
7862
7863 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7864 assert(UseAVX > 0, "requires some form of AVX");
7865 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7866 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7867 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7868 }
7869
7870 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7871 assert(UseAVX > 2, "requires AVX512");
7872 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7873 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7874 attributes.set_is_evex_instruction();
7875 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7876 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
7877 }
7878
7879 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7880 assert(UseAVX > 2, "requires AVX512");
7881 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7882 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7883 attributes.set_is_evex_instruction();
7884 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7885 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7886 }
7887
7888 // logical operations packed integers
7889 void Assembler::pand(XMMRegister dst, XMMRegister src) {
7890 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7891 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7892 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7893 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7894 }
7895
7896 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7897 assert(UseAVX > 0, "requires some form of AVX");
7898 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7899 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7900 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7901 }
7902
7903 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7904 assert(UseAVX > 0, "requires some form of AVX");
7905 InstructionMark im(this);
7906 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7907 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7908 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7909 emit_int8((unsigned char)0xDB);
7910 emit_operand(dst, src, 0);
7911 }
7912
7913 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7914 evpandq(dst, k0, nds, src, false, vector_len);
7915 }
7916
7917 void Assembler::evpandq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7918 evpandq(dst, k0, nds, src, false, vector_len);
7919 }
7920
7921 //Variable Shift packed integers logically left.
7922 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7923 assert(UseAVX > 1, "requires AVX2");
7924 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7925 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7926 emit_int16(0x47, (0xC0 | encode));
7927 }
7928
7929 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7930 assert(UseAVX > 1, "requires AVX2");
7931 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7932 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7933 emit_int16(0x47, (0xC0 | encode));
7934 }
7935
7936 //Variable Shift packed integers logically right.
7937 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7938 assert(UseAVX > 1, "requires AVX2");
7939 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7940 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7941 emit_int16(0x45, (0xC0 | encode));
7942 }
7943
7944 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7945 assert(UseAVX > 1, "requires AVX2");
7946 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7947 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7948 emit_int16(0x45, (0xC0 | encode));
7949 }
7950
7951 //Variable right Shift arithmetic packed integers .
7952 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7953 assert(UseAVX > 1, "requires AVX2");
7954 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7955 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7956 emit_int16(0x46, (0xC0 | encode));
7957 }
7958
7959 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7960 assert(VM_Version::supports_avx512bw(), "");
7961 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7962 attributes.set_is_evex_instruction();
7963 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7964 emit_int16(0x11, (0xC0 | encode));
7965 }
7966
7967 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7968 assert(UseAVX > 2, "requires AVX512");
7969 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
7970 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7971 attributes.set_is_evex_instruction();
7972 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7973 emit_int16(0x46, (0xC0 | encode));
7974 }
7975
7976 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7977 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7978 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7979 attributes.set_is_evex_instruction();
7980 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7981 emit_int16(0x71, (0xC0 | encode));
7982 }
7983
7984 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7985 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7986 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7987 attributes.set_is_evex_instruction();
7988 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7989 emit_int16(0x73, (0xC0 | encode));
7990 }
7991
7992 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
7993 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7994 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7995 attributes.set_rex_vex_w_reverted();
7996 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7997 emit_int16((unsigned char)0xDF, (0xC0 | encode));
7998 }
7999
8000 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8001 assert(UseAVX > 0, "requires some form of AVX");
8002 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8003 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8004 emit_int16((unsigned char)0xDF, (0xC0 | encode));
8005 }
8006
8007 void Assembler::por(XMMRegister dst, XMMRegister src) {
8008 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8009 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8010 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8011 emit_int16((unsigned char)0xEB, (0xC0 | encode));
8012 }
8013
8014 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8015 assert(UseAVX > 0, "requires some form of AVX");
8016 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8017 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8018 emit_int16((unsigned char)0xEB, (0xC0 | encode));
8019 }
8020
8021 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8022 assert(UseAVX > 0, "requires some form of AVX");
8023 InstructionMark im(this);
8024 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8025 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
8026 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8027 emit_int8((unsigned char)0xEB);
8028 emit_operand(dst, src, 0);
8029 }
8030
8031 void Assembler::evporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8032 evporq(dst, k0, nds, src, false, vector_len);
8033 }
8034
8035 void Assembler::evporq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8036 evporq(dst, k0, nds, src, false, vector_len);
8037 }
8038
8039 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8040 assert(VM_Version::supports_evex(), "");
8041 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
8042 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8043 attributes.set_is_evex_instruction();
8044 attributes.set_embedded_opmask_register_specifier(mask);
8045 if (merge) {
8046 attributes.reset_is_clear_context();
8047 }
8048 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8049 emit_int16((unsigned char)0xEB, (0xC0 | encode));
8050 }
8051
8052 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8053 assert(VM_Version::supports_evex(), "");
8054 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
8055 InstructionMark im(this);
8056 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8057 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
8058 attributes.set_is_evex_instruction();
8059 attributes.set_embedded_opmask_register_specifier(mask);
8060 if (merge) {
8061 attributes.reset_is_clear_context();
8062 }
8063 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8064 emit_int8((unsigned char)0xEB);
8065 emit_operand(dst, src, 0);
8066 }
8067
8068 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
8069 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8070 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8071 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8072 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8073 }
8074
8075 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8076 assert(UseAVX > 0, "requires some form of AVX");
8077 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8078 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
8079 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
8080 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8081 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8082 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8083 }
8084
8085 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8086 assert(UseAVX > 0, "requires some form of AVX");
8087 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
8088 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
8089 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
8090 InstructionMark im(this);
8091 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8092 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
8093 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8094 emit_int8((unsigned char)0xEF);
8095 emit_operand(dst, src, 0);
8096 }
8097
8098 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8099 assert(UseAVX > 2, "requires some form of EVEX");
8100 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8101 attributes.set_rex_vex_w_reverted();
8102 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8103 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8104 }
8105
8106 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8107 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
8108 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8109 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8110 attributes.set_is_evex_instruction();
8111 attributes.set_embedded_opmask_register_specifier(mask);
8112 if (merge) {
8113 attributes.reset_is_clear_context();
8114 }
8115 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8116 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8117 }
8118
8119 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8120 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8121 InstructionMark im(this);
8122 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8123 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8124 attributes.set_is_evex_instruction();
8125 attributes.set_embedded_opmask_register_specifier(mask);
8126 if (merge) {
8127 attributes.reset_is_clear_context();
8128 }
8129 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8130 emit_int8((unsigned char)0xEF);
8131 emit_operand(dst, src, 0);
8132 }
8133
8134 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8135 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
8136 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8137 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8138 attributes.set_is_evex_instruction();
8139 attributes.set_embedded_opmask_register_specifier(mask);
8140 if (merge) {
8141 attributes.reset_is_clear_context();
8142 }
8143 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8144 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8145 }
8146
8147 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8148 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8149 InstructionMark im(this);
8150 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8151 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8152 attributes.set_is_evex_instruction();
8153 attributes.set_embedded_opmask_register_specifier(mask);
8154 if (merge) {
8155 attributes.reset_is_clear_context();
8156 }
8157 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8158 emit_int8((unsigned char)0xEF);
8159 emit_operand(dst, src, 0);
8160 }
8161
8162 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8163 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8164 InstructionMark im(this);
8165 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8166 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8167 attributes.set_is_evex_instruction();
8168 attributes.set_embedded_opmask_register_specifier(mask);
8169 if (merge) {
8170 attributes.reset_is_clear_context();
8171 }
8172 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8173 emit_int8((unsigned char)0xDB);
8174 emit_operand(dst, src, 0);
8175 }
8176
8177 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8178 assert(VM_Version::supports_evex(), "requires AVX512F");
8179 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
8180 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8181 attributes.set_is_evex_instruction();
8182 attributes.set_embedded_opmask_register_specifier(mask);
8183 if (merge) {
8184 attributes.reset_is_clear_context();
8185 }
8186 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8187 emit_int16((unsigned char)0xDB, (0xC0 | encode));
8188 }
8189
8190 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8191 assert(VM_Version::supports_evex(), "requires AVX512F");
8192 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
8193 InstructionMark im(this);
8194 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8195 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8196 attributes.set_is_evex_instruction();
8197 attributes.set_embedded_opmask_register_specifier(mask);
8198 if (merge) {
8199 attributes.reset_is_clear_context();
8200 }
8201 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8202 emit_int8((unsigned char)0xDB);
8203 emit_operand(dst, src, 0);
8204 }
8205
8206 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8207 assert(VM_Version::supports_evex(), "requires AVX512F");
8208 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
8209 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8210 attributes.set_is_evex_instruction();
8211 attributes.set_embedded_opmask_register_specifier(mask);
8212 if (merge) {
8213 attributes.reset_is_clear_context();
8214 }
8215 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8216 emit_int16((unsigned char)0xEB, (0xC0 | encode));
8217 }
8218
8219 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8220 assert(VM_Version::supports_evex(), "requires AVX512F");
8221 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
8222 InstructionMark im(this);
8223 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8224 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8225 attributes.set_is_evex_instruction();
8226 attributes.set_embedded_opmask_register_specifier(mask);
8227 if (merge) {
8228 attributes.reset_is_clear_context();
8229 }
8230 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8231 emit_int8((unsigned char)0xEB);
8232 emit_operand(dst, src, 0);
8233 }
8234
8235 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8236 assert(VM_Version::supports_evex(), "requires EVEX support");
8237 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8238 attributes.set_is_evex_instruction();
8239 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8240 emit_int16((unsigned char)0xEF, (0xC0 | encode));
8241 }
8242
8243 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
8244 assert(VM_Version::supports_evex(), "requires EVEX support");
8245 assert(dst != xnoreg, "sanity");
8246 InstructionMark im(this);
8247 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8248 attributes.set_is_evex_instruction();
8249 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8250 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8251 emit_int8((unsigned char)0xEF);
8252 emit_operand(dst, src, 0);
8253 }
8254
8255 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8256 assert(VM_Version::supports_evex(), "requires EVEX support");
8257 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8258 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8259 attributes.set_is_evex_instruction();
8260 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8261 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8262 }
8263
8264 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8265 assert(VM_Version::supports_evex(), "requires EVEX support");
8266 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8267 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8268 attributes.set_is_evex_instruction();
8269 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8270 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8271 }
8272
8273 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8274 assert(VM_Version::supports_evex(), "requires EVEX support");
8275 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8276 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8277 attributes.set_is_evex_instruction();
8278 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8279 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8280 }
8281
8282 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
8283 assert(VM_Version::supports_evex(), "requires EVEX support");
8284 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8285 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8286 attributes.set_is_evex_instruction();
8287 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8288 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
8289 }
8290
8291 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8292 assert(VM_Version::supports_evex(), "requires EVEX support");
8293 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8294 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8295 attributes.set_is_evex_instruction();
8296 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8297 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8298 }
8299
8300 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8301 assert(VM_Version::supports_evex(), "requires EVEX support");
8302 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8303 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8304 attributes.set_is_evex_instruction();
8305 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8306 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8307 }
8308
8309 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8310 assert(VM_Version::supports_evex(), "requires EVEX support");
8311 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8312 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8313 attributes.set_is_evex_instruction();
8314 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8315 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8316 }
8317
8318 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8319 assert(VM_Version::supports_evex(), "requires EVEX support");
8320 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8321 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8322 attributes.set_is_evex_instruction();
8323 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8324 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8325 }
8326
8327 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8328 assert(VM_Version::supports_avx512cd(), "");
8329 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8330 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8331 attributes.set_is_evex_instruction();
8332 attributes.set_embedded_opmask_register_specifier(mask);
8333 if (merge) {
8334 attributes.reset_is_clear_context();
8335 }
8336 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8337 emit_int16(0x44, (0xC0 | encode));
8338 }
8339
8340 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8341 assert(VM_Version::supports_avx512cd(), "");
8342 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8343 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8344 attributes.set_is_evex_instruction();
8345 attributes.set_embedded_opmask_register_specifier(mask);
8346 if (merge) {
8347 attributes.reset_is_clear_context();
8348 }
8349 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8350 emit_int16(0x44, (0xC0 | encode));
8351 }
8352
8353 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8354 assert(VM_Version::supports_evex(), "requires EVEX support");
8355 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8356 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8357 attributes.set_is_evex_instruction();
8358 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8359 emit_int8(0x25);
8360 emit_int8((unsigned char)(0xC0 | encode));
8361 emit_int8(imm8);
8362 }
8363
8364 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
8365 assert(VM_Version::supports_evex(), "requires EVEX support");
8366 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8367 assert(dst != xnoreg, "sanity");
8368 InstructionMark im(this);
8369 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8370 attributes.set_is_evex_instruction();
8371 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8372 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8373 emit_int8(0x25);
8374 emit_operand(dst, src3, 1);
8375 emit_int8(imm8);
8376 }
8377
8378 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8379 assert(VM_Version::supports_evex(), "requires AVX512F");
8380 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
8381 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8382 attributes.set_is_evex_instruction();
8383 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8384 emit_int8(0x25);
8385 emit_int8((unsigned char)(0xC0 | encode));
8386 emit_int8(imm8);
8387 }
8388
8389 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
8390 assert(VM_Version::supports_evex(), "requires EVEX support");
8391 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8392 assert(dst != xnoreg, "sanity");
8393 InstructionMark im(this);
8394 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8395 attributes.set_is_evex_instruction();
8396 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8397 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8398 emit_int8(0x25);
8399 emit_operand(dst, src3, 1);
8400 emit_int8(imm8);
8401 }
8402
8403 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8404 assert(VM_Version::supports_evex(), "");
8405 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8406 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8407 attributes.set_is_evex_instruction();
8408 attributes.set_embedded_opmask_register_specifier(mask);
8409 if (merge) {
8410 attributes.reset_is_clear_context();
8411 }
8412 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8413 emit_int16((unsigned char)0x88, (0xC0 | encode));
8414 }
8415
8416 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8417 assert(VM_Version::supports_evex(), "");
8418 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8419 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8420 attributes.set_is_evex_instruction();
8421 attributes.set_embedded_opmask_register_specifier(mask);
8422 if (merge) {
8423 attributes.reset_is_clear_context();
8424 }
8425 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8426 emit_int16((unsigned char)0x88, (0xC0 | encode));
8427 }
8428
8429 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8430 assert(VM_Version::supports_avx512_vbmi2(), "");
8431 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8432 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8433 attributes.set_is_evex_instruction();
8434 attributes.set_embedded_opmask_register_specifier(mask);
8435 if (merge) {
8436 attributes.reset_is_clear_context();
8437 }
8438 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8439 emit_int16(0x62, (0xC0 | encode));
8440 }
8441
8442 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8443 assert(VM_Version::supports_avx512_vbmi2(), "");
8444 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8445 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8446 attributes.set_is_evex_instruction();
8447 attributes.set_embedded_opmask_register_specifier(mask);
8448 if (merge) {
8449 attributes.reset_is_clear_context();
8450 }
8451 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8452 emit_int16(0x62, (0xC0 | encode));
8453 }
8454
8455 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8456 assert(VM_Version::supports_evex(), "");
8457 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8458 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8459 attributes.set_is_evex_instruction();
8460 attributes.set_embedded_opmask_register_specifier(mask);
8461 if (merge) {
8462 attributes.reset_is_clear_context();
8463 }
8464 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8465 emit_int16((unsigned char)0x89, (0xC0 | encode));
8466 }
8467
8468 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8469 assert(VM_Version::supports_evex(), "");
8470 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8471 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8472 attributes.set_is_evex_instruction();
8473 attributes.set_embedded_opmask_register_specifier(mask);
8474 if (merge) {
8475 attributes.reset_is_clear_context();
8476 }
8477 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8478 emit_int16((unsigned char)0x89, (0xC0 | encode));
8479 }
8480
8481 // vinserti forms
8482
8483 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8484 assert(VM_Version::supports_avx2(), "");
8485 assert(imm8 <= 0x01, "imm8: %u", imm8);
8486 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8487 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8488 // last byte:
8489 // 0x00 - insert into lower 128 bits
8490 // 0x01 - insert into upper 128 bits
8491 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
8492 }
8493
8494 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8495 assert(VM_Version::supports_avx2(), "");
8496 assert(dst != xnoreg, "sanity");
8497 assert(imm8 <= 0x01, "imm8: %u", imm8);
8498 InstructionMark im(this);
8499 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8500 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8501 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8502 emit_int8(0x38);
8503 emit_operand(dst, src, 1);
8504 // 0x00 - insert into lower 128 bits
8505 // 0x01 - insert into upper 128 bits
8506 emit_int8(imm8 & 0x01);
8507 }
8508
8509 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8510 assert(VM_Version::supports_evex(), "");
8511 assert(imm8 <= 0x03, "imm8: %u", imm8);
8512 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8513 attributes.set_is_evex_instruction();
8514 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8515 // imm8:
8516 // 0x00 - insert into q0 128 bits (0..127)
8517 // 0x01 - insert into q1 128 bits (128..255)
8518 // 0x02 - insert into q2 128 bits (256..383)
8519 // 0x03 - insert into q3 128 bits (384..511)
8520 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
8521 }
8522
8523 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8524 assert(VM_Version::supports_evex(), "");
8525 assert(dst != xnoreg, "sanity");
8526 assert(imm8 <= 0x03, "imm8: %u", imm8);
8527 InstructionMark im(this);
8528 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8529 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8530 attributes.set_is_evex_instruction();
8531 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8532 emit_int8(0x18);
8533 emit_operand(dst, src, 1);
8534 // 0x00 - insert into q0 128 bits (0..127)
8535 // 0x01 - insert into q1 128 bits (128..255)
8536 // 0x02 - insert into q2 128 bits (256..383)
8537 // 0x03 - insert into q3 128 bits (384..511)
8538 emit_int8(imm8 & 0x03);
8539 }
8540
8541 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8542 assert(VM_Version::supports_evex(), "");
8543 assert(imm8 <= 0x01, "imm8: %u", imm8);
8544 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8545 attributes.set_is_evex_instruction();
8546 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8547 //imm8:
8548 // 0x00 - insert into lower 256 bits
8549 // 0x01 - insert into upper 256 bits
8550 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
8551 }
8552
8553
8554 // vinsertf forms
8555
8556 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8557 assert(VM_Version::supports_avx(), "");
8558 assert(imm8 <= 0x01, "imm8: %u", imm8);
8559 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8560 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8561 // imm8:
8562 // 0x00 - insert into lower 128 bits
8563 // 0x01 - insert into upper 128 bits
8564 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
8565 }
8566
8567 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8568 assert(VM_Version::supports_avx(), "");
8569 assert(dst != xnoreg, "sanity");
8570 assert(imm8 <= 0x01, "imm8: %u", imm8);
8571 InstructionMark im(this);
8572 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8573 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8574 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8575 emit_int8(0x18);
8576 emit_operand(dst, src, 1);
8577 // 0x00 - insert into lower 128 bits
8578 // 0x01 - insert into upper 128 bits
8579 emit_int8(imm8 & 0x01);
8580 }
8581
8582 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8583 assert(VM_Version::supports_evex(), "");
8584 assert(imm8 <= 0x03, "imm8: %u", imm8);
8585 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8586 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8587 // imm8:
8588 // 0x00 - insert into q0 128 bits (0..127)
8589 // 0x01 - insert into q1 128 bits (128..255)
8590 // 0x02 - insert into q0 128 bits (256..383)
8591 // 0x03 - insert into q1 128 bits (384..512)
8592 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
8593 }
8594
8595 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8596 assert(VM_Version::supports_evex(), "");
8597 assert(dst != xnoreg, "sanity");
8598 assert(imm8 <= 0x03, "imm8: %u", imm8);
8599 InstructionMark im(this);
8600 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8601 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8602 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8603 emit_int8(0x18);
8604 emit_operand(dst, src, 1);
8605 // 0x00 - insert into q0 128 bits (0..127)
8606 // 0x01 - insert into q1 128 bits (128..255)
8607 // 0x02 - insert into q0 128 bits (256..383)
8608 // 0x03 - insert into q1 128 bits (384..512)
8609 emit_int8(imm8 & 0x03);
8610 }
8611
8612 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8613 assert(VM_Version::supports_evex(), "");
8614 assert(imm8 <= 0x01, "imm8: %u", imm8);
8615 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8616 attributes.set_is_evex_instruction();
8617 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8618 // imm8:
8619 // 0x00 - insert into lower 256 bits
8620 // 0x01 - insert into upper 256 bits
8621 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
8622 }
8623
8624 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8625 assert(VM_Version::supports_evex(), "");
8626 assert(dst != xnoreg, "sanity");
8627 assert(imm8 <= 0x01, "imm8: %u", imm8);
8628 InstructionMark im(this);
8629 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8630 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8631 attributes.set_is_evex_instruction();
8632 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8633 emit_int8(0x1A);
8634 emit_operand(dst, src, 1);
8635 // 0x00 - insert into lower 256 bits
8636 // 0x01 - insert into upper 256 bits
8637 emit_int8(imm8 & 0x01);
8638 }
8639
8640
8641 // vextracti forms
8642
8643 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8644 assert(VM_Version::supports_avx2(), "");
8645 assert(imm8 <= 0x01, "imm8: %u", imm8);
8646 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8647 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8648 // imm8:
8649 // 0x00 - extract from lower 128 bits
8650 // 0x01 - extract from upper 128 bits
8651 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
8652 }
8653
8654 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
8655 assert(VM_Version::supports_avx2(), "");
8656 assert(src != xnoreg, "sanity");
8657 assert(imm8 <= 0x01, "imm8: %u", imm8);
8658 InstructionMark im(this);
8659 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8660 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8661 attributes.reset_is_clear_context();
8662 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8663 emit_int8(0x39);
8664 emit_operand(src, dst, 1);
8665 // 0x00 - extract from lower 128 bits
8666 // 0x01 - extract from upper 128 bits
8667 emit_int8(imm8 & 0x01);
8668 }
8669
8670 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8671 assert(VM_Version::supports_evex(), "");
8672 assert(imm8 <= 0x03, "imm8: %u", imm8);
8673 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8674 attributes.set_is_evex_instruction();
8675 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8676 // imm8:
8677 // 0x00 - extract from bits 127:0
8678 // 0x01 - extract from bits 255:128
8679 // 0x02 - extract from bits 383:256
8680 // 0x03 - extract from bits 511:384
8681 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8682 }
8683
8684 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
8685 assert(VM_Version::supports_evex(), "");
8686 assert(src != xnoreg, "sanity");
8687 assert(imm8 <= 0x03, "imm8: %u", imm8);
8688 InstructionMark im(this);
8689 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8690 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8691 attributes.reset_is_clear_context();
8692 attributes.set_is_evex_instruction();
8693 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8694 emit_int8(0x39);
8695 emit_operand(src, dst, 1);
8696 // 0x00 - extract from bits 127:0
8697 // 0x01 - extract from bits 255:128
8698 // 0x02 - extract from bits 383:256
8699 // 0x03 - extract from bits 511:384
8700 emit_int8(imm8 & 0x03);
8701 }
8702
8703 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8704 assert(VM_Version::supports_avx512dq(), "");
8705 assert(imm8 <= 0x03, "imm8: %u", imm8);
8706 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8707 attributes.set_is_evex_instruction();
8708 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8709 // imm8:
8710 // 0x00 - extract from bits 127:0
8711 // 0x01 - extract from bits 255:128
8712 // 0x02 - extract from bits 383:256
8713 // 0x03 - extract from bits 511:384
8714 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8715 }
8716
8717 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8718 assert(VM_Version::supports_evex(), "");
8719 assert(imm8 <= 0x01, "imm8: %u", imm8);
8720 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8721 attributes.set_is_evex_instruction();
8722 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8723 // imm8:
8724 // 0x00 - extract from lower 256 bits
8725 // 0x01 - extract from upper 256 bits
8726 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
8727 }
8728
8729 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
8730 assert(VM_Version::supports_evex(), "");
8731 assert(src != xnoreg, "sanity");
8732 assert(imm8 <= 0x01, "imm8: %u", imm8);
8733 InstructionMark im(this);
8734 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8735 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8736 attributes.reset_is_clear_context();
8737 attributes.set_is_evex_instruction();
8738 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8739 emit_int8(0x38);
8740 emit_operand(src, dst, 1);
8741 // 0x00 - extract from lower 256 bits
8742 // 0x01 - extract from upper 256 bits
8743 emit_int8(imm8 & 0x01);
8744 }
8745 // vextractf forms
8746
8747 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8748 assert(VM_Version::supports_avx(), "");
8749 assert(imm8 <= 0x01, "imm8: %u", imm8);
8750 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8751 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8752 // imm8:
8753 // 0x00 - extract from lower 128 bits
8754 // 0x01 - extract from upper 128 bits
8755 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
8756 }
8757
8758 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
8759 assert(VM_Version::supports_avx(), "");
8760 assert(src != xnoreg, "sanity");
8761 assert(imm8 <= 0x01, "imm8: %u", imm8);
8762 InstructionMark im(this);
8763 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8764 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8765 attributes.reset_is_clear_context();
8766 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8767 emit_int8(0x19);
8768 emit_operand(src, dst, 1);
8769 // 0x00 - extract from lower 128 bits
8770 // 0x01 - extract from upper 128 bits
8771 emit_int8(imm8 & 0x01);
8772 }
8773
8774 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8775 assert(VM_Version::supports_evex(), "");
8776 assert(imm8 <= 0x03, "imm8: %u", imm8);
8777 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8778 attributes.set_is_evex_instruction();
8779 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8780 // imm8:
8781 // 0x00 - extract from bits 127:0
8782 // 0x01 - extract from bits 255:128
8783 // 0x02 - extract from bits 383:256
8784 // 0x03 - extract from bits 511:384
8785 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8786 }
8787
8788 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
8789 assert(VM_Version::supports_evex(), "");
8790 assert(src != xnoreg, "sanity");
8791 assert(imm8 <= 0x03, "imm8: %u", imm8);
8792 InstructionMark im(this);
8793 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8794 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8795 attributes.reset_is_clear_context();
8796 attributes.set_is_evex_instruction();
8797 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8798 emit_int8(0x19);
8799 emit_operand(src, dst, 1);
8800 // 0x00 - extract from bits 127:0
8801 // 0x01 - extract from bits 255:128
8802 // 0x02 - extract from bits 383:256
8803 // 0x03 - extract from bits 511:384
8804 emit_int8(imm8 & 0x03);
8805 }
8806
8807 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8808 assert(VM_Version::supports_avx512dq(), "");
8809 assert(imm8 <= 0x03, "imm8: %u", imm8);
8810 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8811 attributes.set_is_evex_instruction();
8812 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8813 // imm8:
8814 // 0x00 - extract from bits 127:0
8815 // 0x01 - extract from bits 255:128
8816 // 0x02 - extract from bits 383:256
8817 // 0x03 - extract from bits 511:384
8818 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8819 }
8820
8821 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8822 assert(VM_Version::supports_evex(), "");
8823 assert(imm8 <= 0x01, "imm8: %u", imm8);
8824 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8825 attributes.set_is_evex_instruction();
8826 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8827 // imm8:
8828 // 0x00 - extract from lower 256 bits
8829 // 0x01 - extract from upper 256 bits
8830 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
8831 }
8832
8833 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
8834 assert(VM_Version::supports_evex(), "");
8835 assert(src != xnoreg, "sanity");
8836 assert(imm8 <= 0x01, "imm8: %u", imm8);
8837 InstructionMark im(this);
8838 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8839 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
8840 attributes.reset_is_clear_context();
8841 attributes.set_is_evex_instruction();
8842 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8843 emit_int8(0x1B);
8844 emit_operand(src, dst, 1);
8845 // 0x00 - extract from lower 256 bits
8846 // 0x01 - extract from upper 256 bits
8847 emit_int8(imm8 & 0x01);
8848 }
8849
8850 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8851 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
8852 assert(VM_Version::supports_avx2(), "");
8853 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8854 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8855 emit_int16(0x78, (0xC0 | encode));
8856 }
8857
8858 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
8859 assert(VM_Version::supports_avx2(), "");
8860 assert(dst != xnoreg, "sanity");
8861 InstructionMark im(this);
8862 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8863 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
8864 // swap src<->dst for encoding
8865 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8866 emit_int8(0x78);
8867 emit_operand(dst, src, 0);
8868 }
8869
8870 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8871 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
8872 assert(VM_Version::supports_avx2(), "");
8873 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8874 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8875 emit_int16(0x79, (0xC0 | encode));
8876 }
8877
8878 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
8879 assert(VM_Version::supports_avx2(), "");
8880 assert(dst != xnoreg, "sanity");
8881 InstructionMark im(this);
8882 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8883 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
8884 // swap src<->dst for encoding
8885 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8886 emit_int8(0x79);
8887 emit_operand(dst, src, 0);
8888 }
8889
8890 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8891 assert(UseAVX > 0, "requires some form of AVX");
8892 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8893 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8894 emit_int16((unsigned char)0xF6, (0xC0 | encode));
8895 }
8896
8897 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8898 assert(UseAVX > 0, "requires some form of AVX");
8899 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8900 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8901 emit_int16(0x69, (0xC0 | encode));
8902 }
8903
8904 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8905 assert(UseAVX > 0, "requires some form of AVX");
8906 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8907 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8908 emit_int16(0x61, (0xC0 | encode));
8909 }
8910
8911 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8912 assert(UseAVX > 0, "requires some form of AVX");
8913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8914 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8915 emit_int16(0x6A, (0xC0 | encode));
8916 }
8917
8918 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8919 assert(UseAVX > 0, "requires some form of AVX");
8920 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8921 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8922 emit_int16(0x62, (0xC0 | encode));
8923 }
8924
8925 // xmm/mem sourced byte/word/dword/qword replicate
8926 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8927 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8928 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8929 attributes.set_is_evex_instruction();
8930 attributes.set_embedded_opmask_register_specifier(mask);
8931 if (merge) {
8932 attributes.reset_is_clear_context();
8933 }
8934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8935 emit_int16((unsigned char)0xFC, (0xC0 | encode));
8936 }
8937
8938 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8939 InstructionMark im(this);
8940 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8941 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8942 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8943 attributes.set_is_evex_instruction();
8944 attributes.set_embedded_opmask_register_specifier(mask);
8945 if (merge) {
8946 attributes.reset_is_clear_context();
8947 }
8948 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8949 emit_int8((unsigned char)0xFC);
8950 emit_operand(dst, src, 0);
8951 }
8952
8953 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8954 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8955 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8956 attributes.set_is_evex_instruction();
8957 attributes.set_embedded_opmask_register_specifier(mask);
8958 if (merge) {
8959 attributes.reset_is_clear_context();
8960 }
8961 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8962 emit_int16((unsigned char)0xFD, (0xC0 | encode));
8963 }
8964
8965 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8966 InstructionMark im(this);
8967 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8968 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8969 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8970 attributes.set_is_evex_instruction();
8971 attributes.set_embedded_opmask_register_specifier(mask);
8972 if (merge) {
8973 attributes.reset_is_clear_context();
8974 }
8975 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8976 emit_int8((unsigned char)0xFD);
8977 emit_operand(dst, src, 0);
8978 }
8979
8980 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8981 assert(VM_Version::supports_evex(), "");
8982 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8983 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8984 attributes.set_is_evex_instruction();
8985 attributes.set_embedded_opmask_register_specifier(mask);
8986 if (merge) {
8987 attributes.reset_is_clear_context();
8988 }
8989 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8990 emit_int16((unsigned char)0xFE, (0xC0 | encode));
8991 }
8992
8993 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8994 InstructionMark im(this);
8995 assert(VM_Version::supports_evex(), "");
8996 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8997 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8998 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8999 attributes.set_is_evex_instruction();
9000 attributes.set_embedded_opmask_register_specifier(mask);
9001 if (merge) {
9002 attributes.reset_is_clear_context();
9003 }
9004 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9005 emit_int8((unsigned char)0xFE);
9006 emit_operand(dst, src, 0);
9007 }
9008
9009 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9010 assert(VM_Version::supports_evex(), "");
9011 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9012 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9013 attributes.set_is_evex_instruction();
9014 attributes.set_embedded_opmask_register_specifier(mask);
9015 if (merge) {
9016 attributes.reset_is_clear_context();
9017 }
9018 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9019 emit_int16((unsigned char)0xD4, (0xC0 | encode));
9020 }
9021
9022 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9023 InstructionMark im(this);
9024 assert(VM_Version::supports_evex(), "");
9025 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9026 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9027 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9028 attributes.set_is_evex_instruction();
9029 attributes.set_embedded_opmask_register_specifier(mask);
9030 if (merge) {
9031 attributes.reset_is_clear_context();
9032 }
9033 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9034 emit_int8((unsigned char)0xD4);
9035 emit_operand(dst, src, 0);
9036 }
9037
9038 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9039 assert(VM_Version::supports_evex(), "");
9040 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9041 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9042 attributes.set_is_evex_instruction();
9043 attributes.set_embedded_opmask_register_specifier(mask);
9044 if (merge) {
9045 attributes.reset_is_clear_context();
9046 }
9047 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9048 emit_int16(0x58, (0xC0 | encode));
9049 }
9050
9051 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9052 InstructionMark im(this);
9053 assert(VM_Version::supports_evex(), "");
9054 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9055 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9056 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9057 attributes.set_is_evex_instruction();
9058 attributes.set_embedded_opmask_register_specifier(mask);
9059 if (merge) {
9060 attributes.reset_is_clear_context();
9061 }
9062 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9063 emit_int8(0x58);
9064 emit_operand(dst, src, 0);
9065 }
9066
9067 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9068 assert(VM_Version::supports_evex(), "");
9069 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9070 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9071 attributes.set_is_evex_instruction();
9072 attributes.set_embedded_opmask_register_specifier(mask);
9073 if (merge) {
9074 attributes.reset_is_clear_context();
9075 }
9076 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9077 emit_int16(0x58, (0xC0 | encode));
9078 }
9079
9080 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9081 InstructionMark im(this);
9082 assert(VM_Version::supports_evex(), "");
9083 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9084 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9085 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9086 attributes.set_is_evex_instruction();
9087 attributes.set_embedded_opmask_register_specifier(mask);
9088 if (merge) {
9089 attributes.reset_is_clear_context();
9090 }
9091 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9092 emit_int8(0x58);
9093 emit_operand(dst, src, 0);
9094 }
9095
9096 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9097 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9098 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9099 attributes.set_is_evex_instruction();
9100 attributes.set_embedded_opmask_register_specifier(mask);
9101 if (merge) {
9102 attributes.reset_is_clear_context();
9103 }
9104 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9105 emit_int16((unsigned char)0xF8, (0xC0 | encode));
9106 }
9107
9108 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9109 InstructionMark im(this);
9110 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9111 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9112 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9113 attributes.set_is_evex_instruction();
9114 attributes.set_embedded_opmask_register_specifier(mask);
9115 if (merge) {
9116 attributes.reset_is_clear_context();
9117 }
9118 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9119 emit_int8((unsigned char)0xF8);
9120 emit_operand(dst, src, 0);
9121 }
9122
9123 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9124 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9126 attributes.set_is_evex_instruction();
9127 attributes.set_embedded_opmask_register_specifier(mask);
9128 if (merge) {
9129 attributes.reset_is_clear_context();
9130 }
9131 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9132 emit_int16((unsigned char)0xF9, (0xC0 | encode));
9133 }
9134
9135 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9136 InstructionMark im(this);
9137 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9138 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9139 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9140 attributes.set_is_evex_instruction();
9141 attributes.set_embedded_opmask_register_specifier(mask);
9142 if (merge) {
9143 attributes.reset_is_clear_context();
9144 }
9145 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9146 emit_int8((unsigned char)0xF9);
9147 emit_operand(dst, src, 0);
9148 }
9149
9150 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9151 assert(VM_Version::supports_evex(), "");
9152 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9153 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9154 attributes.set_is_evex_instruction();
9155 attributes.set_embedded_opmask_register_specifier(mask);
9156 if (merge) {
9157 attributes.reset_is_clear_context();
9158 }
9159 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9160 emit_int16((unsigned char)0xFA, (0xC0 | encode));
9161 }
9162
9163 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9164 InstructionMark im(this);
9165 assert(VM_Version::supports_evex(), "");
9166 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9167 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9168 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9169 attributes.set_is_evex_instruction();
9170 attributes.set_embedded_opmask_register_specifier(mask);
9171 if (merge) {
9172 attributes.reset_is_clear_context();
9173 }
9174 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9175 emit_int8((unsigned char)0xFA);
9176 emit_operand(dst, src, 0);
9177 }
9178
9179 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9180 assert(VM_Version::supports_evex(), "");
9181 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9182 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9183 attributes.set_is_evex_instruction();
9184 attributes.set_embedded_opmask_register_specifier(mask);
9185 if (merge) {
9186 attributes.reset_is_clear_context();
9187 }
9188 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9189 emit_int16((unsigned char)0xFB, (0xC0 | encode));
9190 }
9191
9192 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9193 InstructionMark im(this);
9194 assert(VM_Version::supports_evex(), "");
9195 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9196 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9197 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9198 attributes.set_is_evex_instruction();
9199 attributes.set_embedded_opmask_register_specifier(mask);
9200 if (merge) {
9201 attributes.reset_is_clear_context();
9202 }
9203 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9204 emit_int8((unsigned char)0xFB);
9205 emit_operand(dst, src, 0);
9206 }
9207
9208 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9209 assert(VM_Version::supports_evex(), "");
9210 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9211 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9212 attributes.set_is_evex_instruction();
9213 attributes.set_embedded_opmask_register_specifier(mask);
9214 if (merge) {
9215 attributes.reset_is_clear_context();
9216 }
9217 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9218 emit_int16(0x5C, (0xC0 | encode));
9219 }
9220
9221 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9222 InstructionMark im(this);
9223 assert(VM_Version::supports_evex(), "");
9224 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9225 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9226 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9227 attributes.set_is_evex_instruction();
9228 attributes.set_embedded_opmask_register_specifier(mask);
9229 if (merge) {
9230 attributes.reset_is_clear_context();
9231 }
9232 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9233 emit_int8(0x5C);
9234 emit_operand(dst, src, 0);
9235 }
9236
9237 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9238 assert(VM_Version::supports_evex(), "");
9239 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9240 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9241 attributes.set_is_evex_instruction();
9242 attributes.set_embedded_opmask_register_specifier(mask);
9243 if (merge) {
9244 attributes.reset_is_clear_context();
9245 }
9246 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9247 emit_int16(0x5C, (0xC0 | encode));
9248 }
9249
9250 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9251 InstructionMark im(this);
9252 assert(VM_Version::supports_evex(), "");
9253 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9254 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9255 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9256 attributes.set_is_evex_instruction();
9257 attributes.set_embedded_opmask_register_specifier(mask);
9258 if (merge) {
9259 attributes.reset_is_clear_context();
9260 }
9261 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9262 emit_int8(0x5C);
9263 emit_operand(dst, src, 0);
9264 }
9265
9266 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9267 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9268 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9269 attributes.set_is_evex_instruction();
9270 attributes.set_embedded_opmask_register_specifier(mask);
9271 if (merge) {
9272 attributes.reset_is_clear_context();
9273 }
9274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9275 emit_int16((unsigned char)0xD5, (0xC0 | encode));
9276 }
9277
9278 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9279 InstructionMark im(this);
9280 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9281 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9282 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9283 attributes.set_is_evex_instruction();
9284 attributes.set_embedded_opmask_register_specifier(mask);
9285 if (merge) {
9286 attributes.reset_is_clear_context();
9287 }
9288 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9289 emit_int8((unsigned char)0xD5);
9290 emit_operand(dst, src, 0);
9291 }
9292
9293 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9294 assert(VM_Version::supports_evex(), "");
9295 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9296 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9297 attributes.set_is_evex_instruction();
9298 attributes.set_embedded_opmask_register_specifier(mask);
9299 if (merge) {
9300 attributes.reset_is_clear_context();
9301 }
9302 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9303 emit_int16(0x40, (0xC0 | encode));
9304 }
9305
9306 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9307 InstructionMark im(this);
9308 assert(VM_Version::supports_evex(), "");
9309 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9310 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9311 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9312 attributes.set_is_evex_instruction();
9313 attributes.set_embedded_opmask_register_specifier(mask);
9314 if (merge) {
9315 attributes.reset_is_clear_context();
9316 }
9317 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9318 emit_int8(0x40);
9319 emit_operand(dst, src, 0);
9320 }
9321
9322 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9323 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9324 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9325 attributes.set_is_evex_instruction();
9326 attributes.set_embedded_opmask_register_specifier(mask);
9327 if (merge) {
9328 attributes.reset_is_clear_context();
9329 }
9330 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9331 emit_int16(0x40, (0xC0 | encode));
9332 }
9333
9334 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9335 InstructionMark im(this);
9336 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9337 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9338 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9339 attributes.set_is_evex_instruction();
9340 attributes.set_embedded_opmask_register_specifier(mask);
9341 if (merge) {
9342 attributes.reset_is_clear_context();
9343 }
9344 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9345 emit_int8(0x40);
9346 emit_operand(dst, src, 0);
9347 }
9348
9349 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9350 assert(VM_Version::supports_evex(), "");
9351 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9352 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9353 attributes.set_is_evex_instruction();
9354 attributes.set_embedded_opmask_register_specifier(mask);
9355 if (merge) {
9356 attributes.reset_is_clear_context();
9357 }
9358 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9359 emit_int16(0x59, (0xC0 | encode));
9360 }
9361
9362 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9363 InstructionMark im(this);
9364 assert(VM_Version::supports_evex(), "");
9365 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9366 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9367 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9368 attributes.set_is_evex_instruction();
9369 attributes.set_embedded_opmask_register_specifier(mask);
9370 if (merge) {
9371 attributes.reset_is_clear_context();
9372 }
9373 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9374 emit_int8(0x59);
9375 emit_operand(dst, src, 0);
9376 }
9377
9378 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9379 assert(VM_Version::supports_evex(), "");
9380 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9381 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9382 attributes.set_is_evex_instruction();
9383 attributes.set_embedded_opmask_register_specifier(mask);
9384 if (merge) {
9385 attributes.reset_is_clear_context();
9386 }
9387 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9388 emit_int16(0x59, (0xC0 | encode));
9389 }
9390
9391 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9392 InstructionMark im(this);
9393 assert(VM_Version::supports_evex(), "");
9394 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9395 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9396 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9397 attributes.set_is_evex_instruction();
9398 attributes.set_embedded_opmask_register_specifier(mask);
9399 if (merge) {
9400 attributes.reset_is_clear_context();
9401 }
9402 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9403 emit_int8(0x59);
9404 emit_operand(dst, src, 0);
9405 }
9406
9407 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9408 assert(VM_Version::supports_evex(), "");
9409 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9410 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9411 attributes.set_is_evex_instruction();
9412 attributes.set_embedded_opmask_register_specifier(mask);
9413 if (merge) {
9414 attributes.reset_is_clear_context();
9415 }
9416 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9417 emit_int16(0x51, (0xC0 | encode));
9418 }
9419
9420 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9421 InstructionMark im(this);
9422 assert(VM_Version::supports_evex(), "");
9423 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9424 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9425 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9426 attributes.set_is_evex_instruction();
9427 attributes.set_embedded_opmask_register_specifier(mask);
9428 if (merge) {
9429 attributes.reset_is_clear_context();
9430 }
9431 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9432 emit_int8(0x51);
9433 emit_operand(dst, src, 0);
9434 }
9435
9436 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9437 assert(VM_Version::supports_evex(), "");
9438 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9439 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9440 attributes.set_is_evex_instruction();
9441 attributes.set_embedded_opmask_register_specifier(mask);
9442 if (merge) {
9443 attributes.reset_is_clear_context();
9444 }
9445 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9446 emit_int16(0x51, (0xC0 | encode));
9447 }
9448
9449 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9450 InstructionMark im(this);
9451 assert(VM_Version::supports_evex(), "");
9452 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9453 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9454 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9455 attributes.set_is_evex_instruction();
9456 attributes.set_embedded_opmask_register_specifier(mask);
9457 if (merge) {
9458 attributes.reset_is_clear_context();
9459 }
9460 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9461 emit_int8(0x51);
9462 emit_operand(dst, src, 0);
9463 }
9464
9465
9466 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9467 assert(VM_Version::supports_evex(), "");
9468 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9469 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9470 attributes.set_is_evex_instruction();
9471 attributes.set_embedded_opmask_register_specifier(mask);
9472 if (merge) {
9473 attributes.reset_is_clear_context();
9474 }
9475 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9476 emit_int16(0x5E, (0xC0 | encode));
9477 }
9478
9479 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9480 InstructionMark im(this);
9481 assert(VM_Version::supports_evex(), "");
9482 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9483 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9484 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9485 attributes.set_is_evex_instruction();
9486 attributes.set_embedded_opmask_register_specifier(mask);
9487 if (merge) {
9488 attributes.reset_is_clear_context();
9489 }
9490 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9491 emit_int8(0x5E);
9492 emit_operand(dst, src, 0);
9493 }
9494
9495 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9496 assert(VM_Version::supports_evex(), "");
9497 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9498 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9499 attributes.set_is_evex_instruction();
9500 attributes.set_embedded_opmask_register_specifier(mask);
9501 if (merge) {
9502 attributes.reset_is_clear_context();
9503 }
9504 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9505 emit_int16(0x5E, (0xC0 | encode));
9506 }
9507
9508 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9509 InstructionMark im(this);
9510 assert(VM_Version::supports_evex(), "");
9511 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9512 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9513 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9514 attributes.set_is_evex_instruction();
9515 attributes.set_embedded_opmask_register_specifier(mask);
9516 if (merge) {
9517 attributes.reset_is_clear_context();
9518 }
9519 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9520 emit_int8(0x5E);
9521 emit_operand(dst, src, 0);
9522 }
9523
9524 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9525 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9526 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9527 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9528 attributes.set_is_evex_instruction();
9529 attributes.set_embedded_opmask_register_specifier(mask);
9530 if (merge) {
9531 attributes.reset_is_clear_context();
9532 }
9533 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9534 emit_int16(0x1C, (0xC0 | encode));
9535 }
9536
9537
9538 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9539 InstructionMark im(this);
9540 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9541 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9542 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9543 attributes.set_is_evex_instruction();
9544 attributes.set_embedded_opmask_register_specifier(mask);
9545 if (merge) {
9546 attributes.reset_is_clear_context();
9547 }
9548 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9549 emit_int8(0x1C);
9550 emit_operand(dst, src, 0);
9551 }
9552
9553 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9554 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9555 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9556 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9557 attributes.set_is_evex_instruction();
9558 attributes.set_embedded_opmask_register_specifier(mask);
9559 if (merge) {
9560 attributes.reset_is_clear_context();
9561 }
9562 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9563 emit_int16(0x1D, (0xC0 | encode));
9564 }
9565
9566
9567 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9568 InstructionMark im(this);
9569 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9570 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9571 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9572 attributes.set_is_evex_instruction();
9573 attributes.set_embedded_opmask_register_specifier(mask);
9574 if (merge) {
9575 attributes.reset_is_clear_context();
9576 }
9577 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9578 emit_int8(0x1D);
9579 emit_operand(dst, src, 0);
9580 }
9581
9582 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9583 assert(VM_Version::supports_evex(), "");
9584 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9585 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9586 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9587 attributes.set_is_evex_instruction();
9588 attributes.set_embedded_opmask_register_specifier(mask);
9589 if (merge) {
9590 attributes.reset_is_clear_context();
9591 }
9592 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9593 emit_int16(0x1E, (0xC0 | encode));
9594 }
9595
9596
9597 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9598 InstructionMark im(this);
9599 assert(VM_Version::supports_evex(), "");
9600 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9601 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9602 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9603 attributes.set_is_evex_instruction();
9604 attributes.set_embedded_opmask_register_specifier(mask);
9605 if (merge) {
9606 attributes.reset_is_clear_context();
9607 }
9608 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9609 emit_int8(0x1E);
9610 emit_operand(dst, src, 0);
9611 }
9612
9613 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9614 assert(VM_Version::supports_evex(), "");
9615 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9616 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9617 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9618 attributes.set_is_evex_instruction();
9619 attributes.set_embedded_opmask_register_specifier(mask);
9620 if (merge) {
9621 attributes.reset_is_clear_context();
9622 }
9623 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9624 emit_int16(0x1F, (0xC0 | encode));
9625 }
9626
9627
9628 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9629 InstructionMark im(this);
9630 assert(VM_Version::supports_evex(), "");
9631 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9632 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9633 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9634 attributes.set_is_evex_instruction();
9635 attributes.set_embedded_opmask_register_specifier(mask);
9636 if (merge) {
9637 attributes.reset_is_clear_context();
9638 }
9639 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9640 emit_int8(0x1F);
9641 emit_operand(dst, src, 0);
9642 }
9643
9644 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9645 assert(VM_Version::supports_evex(), "");
9646 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9647 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9648 attributes.set_is_evex_instruction();
9649 attributes.set_embedded_opmask_register_specifier(mask);
9650 if (merge) {
9651 attributes.reset_is_clear_context();
9652 }
9653 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9654 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9655 }
9656
9657 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9658 InstructionMark im(this);
9659 assert(VM_Version::supports_evex(), "");
9660 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9661 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9662 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9663 attributes.set_is_evex_instruction();
9664 attributes.set_embedded_opmask_register_specifier(mask);
9665 if (merge) {
9666 attributes.reset_is_clear_context();
9667 }
9668 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9669 emit_int8((unsigned char)0xA8);
9670 emit_operand(dst, src, 0);
9671 }
9672
9673 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9674 assert(VM_Version::supports_evex(), "");
9675 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9676 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9677 attributes.set_is_evex_instruction();
9678 attributes.set_embedded_opmask_register_specifier(mask);
9679 if (merge) {
9680 attributes.reset_is_clear_context();
9681 }
9682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9683 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9684 }
9685
9686 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9687 InstructionMark im(this);
9688 assert(VM_Version::supports_evex(), "");
9689 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9690 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9691 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9692 attributes.set_is_evex_instruction();
9693 attributes.set_embedded_opmask_register_specifier(mask);
9694 if (merge) {
9695 attributes.reset_is_clear_context();
9696 }
9697 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9698 emit_int8((unsigned char)0xA8);
9699 emit_operand(dst, src, 0);
9700 }
9701
9702 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9703 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9704 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9705 attributes.set_is_evex_instruction();
9706 attributes.set_embedded_opmask_register_specifier(mask);
9707 if (merge) {
9708 attributes.reset_is_clear_context();
9709 }
9710 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9711 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9712 }
9713
9714 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9715 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9716 InstructionMark im(this);
9717 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9718 attributes.set_is_evex_instruction();
9719 attributes.set_embedded_opmask_register_specifier(mask);
9720 if (merge) {
9721 attributes.reset_is_clear_context();
9722 }
9723 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9724 emit_int8((unsigned char)0x8D);
9725 emit_operand(dst, src, 0);
9726 }
9727
9728 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9729 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9730 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9731 attributes.set_is_evex_instruction();
9732 attributes.set_embedded_opmask_register_specifier(mask);
9733 if (merge) {
9734 attributes.reset_is_clear_context();
9735 }
9736 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9737 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9738 }
9739
9740 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9741 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9742 InstructionMark im(this);
9743 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9744 attributes.set_is_evex_instruction();
9745 attributes.set_embedded_opmask_register_specifier(mask);
9746 if (merge) {
9747 attributes.reset_is_clear_context();
9748 }
9749 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9750 emit_int8((unsigned char)0x8D);
9751 emit_operand(dst, src, 0);
9752 }
9753
9754 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9755 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9756 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9757 attributes.set_is_evex_instruction();
9758 attributes.set_embedded_opmask_register_specifier(mask);
9759 if (merge) {
9760 attributes.reset_is_clear_context();
9761 }
9762 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9763 emit_int16(0x36, (0xC0 | encode));
9764 }
9765
9766 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9767 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9768 InstructionMark im(this);
9769 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9770 attributes.set_is_evex_instruction();
9771 attributes.set_embedded_opmask_register_specifier(mask);
9772 if (merge) {
9773 attributes.reset_is_clear_context();
9774 }
9775 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9776 emit_int8(0x36);
9777 emit_operand(dst, src, 0);
9778 }
9779
9780 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9781 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9782 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9783 attributes.set_is_evex_instruction();
9784 attributes.set_embedded_opmask_register_specifier(mask);
9785 if (merge) {
9786 attributes.reset_is_clear_context();
9787 }
9788 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9789 emit_int16(0x36, (0xC0 | encode));
9790 }
9791
9792 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9793 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9794 InstructionMark im(this);
9795 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9796 attributes.set_is_evex_instruction();
9797 attributes.set_embedded_opmask_register_specifier(mask);
9798 if (merge) {
9799 attributes.reset_is_clear_context();
9800 }
9801 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9802 emit_int8(0x36);
9803 emit_operand(dst, src, 0);
9804 }
9805
9806 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9807 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9808 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9809 attributes.set_is_evex_instruction();
9810 attributes.set_embedded_opmask_register_specifier(mask);
9811 if (merge) {
9812 attributes.reset_is_clear_context();
9813 }
9814 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9815 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9816 }
9817
9818 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9819 assert(VM_Version::supports_evex(), "");
9820 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9821 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9822 attributes.set_is_evex_instruction();
9823 attributes.set_embedded_opmask_register_specifier(mask);
9824 if (merge) {
9825 attributes.reset_is_clear_context();
9826 }
9827 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9828 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9829 }
9830
9831 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9832 assert(VM_Version::supports_evex(), "");
9833 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9834 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9835 attributes.set_is_evex_instruction();
9836 attributes.set_embedded_opmask_register_specifier(mask);
9837 if (merge) {
9838 attributes.reset_is_clear_context();
9839 }
9840 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9841 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9842 }
9843
9844 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9845 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9846 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9847 attributes.set_is_evex_instruction();
9848 attributes.set_embedded_opmask_register_specifier(mask);
9849 if (merge) {
9850 attributes.reset_is_clear_context();
9851 }
9852 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9853 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9854 }
9855
9856 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9857 assert(VM_Version::supports_evex(), "");
9858 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9859 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9860 attributes.set_is_evex_instruction();
9861 attributes.set_embedded_opmask_register_specifier(mask);
9862 if (merge) {
9863 attributes.reset_is_clear_context();
9864 }
9865 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9866 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9867 }
9868
9869 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9870 assert(VM_Version::supports_evex(), "");
9871 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9872 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9873 attributes.set_is_evex_instruction();
9874 attributes.set_embedded_opmask_register_specifier(mask);
9875 if (merge) {
9876 attributes.reset_is_clear_context();
9877 }
9878 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9879 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9880 }
9881
9882 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9883 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9884 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9885 attributes.set_is_evex_instruction();
9886 attributes.set_embedded_opmask_register_specifier(mask);
9887 if (merge) {
9888 attributes.reset_is_clear_context();
9889 }
9890 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9891 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9892 }
9893
9894 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9895 assert(VM_Version::supports_evex(), "");
9896 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9897 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9898 attributes.set_is_evex_instruction();
9899 attributes.set_embedded_opmask_register_specifier(mask);
9900 if (merge) {
9901 attributes.reset_is_clear_context();
9902 }
9903 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9904 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9905 }
9906
9907 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9908 assert(VM_Version::supports_evex(), "");
9909 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9910 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9911 attributes.set_is_evex_instruction();
9912 attributes.set_embedded_opmask_register_specifier(mask);
9913 if (merge) {
9914 attributes.reset_is_clear_context();
9915 }
9916 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9917 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9918 }
9919
9920 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9921 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9922 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9923 attributes.set_is_evex_instruction();
9924 attributes.set_embedded_opmask_register_specifier(mask);
9925 if (merge) {
9926 attributes.reset_is_clear_context();
9927 }
9928 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9929 emit_int16((unsigned char)0xF1, (0xC0 | encode));
9930 }
9931
9932 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9933 assert(VM_Version::supports_evex(), "");
9934 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9935 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9936 attributes.set_is_evex_instruction();
9937 attributes.set_embedded_opmask_register_specifier(mask);
9938 if (merge) {
9939 attributes.reset_is_clear_context();
9940 }
9941 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9942 emit_int16((unsigned char)0xF2, (0xC0 | encode));
9943 }
9944
9945 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9946 assert(VM_Version::supports_evex(), "");
9947 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9948 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9949 attributes.set_is_evex_instruction();
9950 attributes.set_embedded_opmask_register_specifier(mask);
9951 if (merge) {
9952 attributes.reset_is_clear_context();
9953 }
9954 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9955 emit_int16((unsigned char)0xF3, (0xC0 | encode));
9956 }
9957
9958 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9959 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9960 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9961 attributes.set_is_evex_instruction();
9962 attributes.set_embedded_opmask_register_specifier(mask);
9963 if (merge) {
9964 attributes.reset_is_clear_context();
9965 }
9966 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9967 emit_int16((unsigned char)0xD1, (0xC0 | encode));
9968 }
9969
9970 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9971 assert(VM_Version::supports_evex(), "");
9972 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9973 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9974 attributes.set_is_evex_instruction();
9975 attributes.set_embedded_opmask_register_specifier(mask);
9976 if (merge) {
9977 attributes.reset_is_clear_context();
9978 }
9979 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9980 emit_int16((unsigned char)0xD2, (0xC0 | encode));
9981 }
9982
9983 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9984 assert(VM_Version::supports_evex(), "");
9985 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9986 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9987 attributes.set_is_evex_instruction();
9988 attributes.set_embedded_opmask_register_specifier(mask);
9989 if (merge) {
9990 attributes.reset_is_clear_context();
9991 }
9992 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9993 emit_int16((unsigned char)0xD3, (0xC0 | encode));
9994 }
9995
9996 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9997 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9998 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9999 attributes.set_is_evex_instruction();
10000 attributes.set_embedded_opmask_register_specifier(mask);
10001 if (merge) {
10002 attributes.reset_is_clear_context();
10003 }
10004 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10005 emit_int16((unsigned char)0xE1, (0xC0 | encode));
10006 }
10007
10008 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10009 assert(VM_Version::supports_evex(), "");
10010 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10011 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10012 attributes.set_is_evex_instruction();
10013 attributes.set_embedded_opmask_register_specifier(mask);
10014 if (merge) {
10015 attributes.reset_is_clear_context();
10016 }
10017 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10018 emit_int16((unsigned char)0xE2, (0xC0 | encode));
10019 }
10020
10021 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10022 assert(VM_Version::supports_evex(), "");
10023 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10024 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10025 attributes.set_is_evex_instruction();
10026 attributes.set_embedded_opmask_register_specifier(mask);
10027 if (merge) {
10028 attributes.reset_is_clear_context();
10029 }
10030 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10031 emit_int16((unsigned char)0xE2, (0xC0 | encode));
10032 }
10033
10034 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10035 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10036 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10037 attributes.set_is_evex_instruction();
10038 attributes.set_embedded_opmask_register_specifier(mask);
10039 if (merge) {
10040 attributes.reset_is_clear_context();
10041 }
10042 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10043 emit_int16(0x12, (0xC0 | encode));
10044 }
10045
10046 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10047 assert(VM_Version::supports_evex(), "");
10048 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10049 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10050 attributes.set_is_evex_instruction();
10051 attributes.set_embedded_opmask_register_specifier(mask);
10052 if (merge) {
10053 attributes.reset_is_clear_context();
10054 }
10055 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10056 emit_int16(0x47, (0xC0 | encode));
10057 }
10058
10059 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10060 assert(VM_Version::supports_evex(), "");
10061 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10062 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10063 attributes.set_is_evex_instruction();
10064 attributes.set_embedded_opmask_register_specifier(mask);
10065 if (merge) {
10066 attributes.reset_is_clear_context();
10067 }
10068 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10069 emit_int16(0x47, (0xC0 | encode));
10070 }
10071
10072 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10073 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10074 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10075 attributes.set_is_evex_instruction();
10076 attributes.set_embedded_opmask_register_specifier(mask);
10077 if (merge) {
10078 attributes.reset_is_clear_context();
10079 }
10080 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10081 emit_int16(0x10, (0xC0 | encode));
10082 }
10083
10084 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10085 assert(VM_Version::supports_evex(), "");
10086 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10087 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10088 attributes.set_is_evex_instruction();
10089 attributes.set_embedded_opmask_register_specifier(mask);
10090 if (merge) {
10091 attributes.reset_is_clear_context();
10092 }
10093 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10094 emit_int16(0x45, (0xC0 | encode));
10095 }
10096
10097 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10098 assert(VM_Version::supports_evex(), "");
10099 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10100 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10101 attributes.set_is_evex_instruction();
10102 attributes.set_embedded_opmask_register_specifier(mask);
10103 if (merge) {
10104 attributes.reset_is_clear_context();
10105 }
10106 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10107 emit_int16(0x45, (0xC0 | encode));
10108 }
10109
10110 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10111 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10112 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10113 attributes.set_is_evex_instruction();
10114 attributes.set_embedded_opmask_register_specifier(mask);
10115 if (merge) {
10116 attributes.reset_is_clear_context();
10117 }
10118 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10119 emit_int16(0x11, (0xC0 | encode));
10120 }
10121
10122 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10123 assert(VM_Version::supports_evex(), "");
10124 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10125 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10126 attributes.set_is_evex_instruction();
10127 attributes.set_embedded_opmask_register_specifier(mask);
10128 if (merge) {
10129 attributes.reset_is_clear_context();
10130 }
10131 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10132 emit_int16(0x46, (0xC0 | encode));
10133 }
10134
10135 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10136 assert(VM_Version::supports_evex(), "");
10137 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10138 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10139 attributes.set_is_evex_instruction();
10140 attributes.set_embedded_opmask_register_specifier(mask);
10141 if (merge) {
10142 attributes.reset_is_clear_context();
10143 }
10144 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10145 emit_int16(0x46, (0xC0 | encode));
10146 }
10147
10148 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10149 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10150 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10151 attributes.set_is_evex_instruction();
10152 attributes.set_embedded_opmask_register_specifier(mask);
10153 if (merge) {
10154 attributes.reset_is_clear_context();
10155 }
10156 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10157 emit_int16(0x38, (0xC0 | encode));
10158 }
10159
10160 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10161 assert(VM_Version::supports_avx512bw(), "");
10162 InstructionMark im(this);
10163 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10164 attributes.set_is_evex_instruction();
10165 attributes.set_embedded_opmask_register_specifier(mask);
10166 if (merge) {
10167 attributes.reset_is_clear_context();
10168 }
10169 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10170 emit_int8(0x38);
10171 emit_operand(dst, src, 0);
10172 }
10173
10174 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10175 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10176 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10177 attributes.set_is_evex_instruction();
10178 attributes.set_embedded_opmask_register_specifier(mask);
10179 if (merge) {
10180 attributes.reset_is_clear_context();
10181 }
10182 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10183 emit_int16((unsigned char)0xEA, (0xC0 | encode));
10184 }
10185
10186 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10187 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10188 InstructionMark im(this);
10189 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10190 attributes.set_is_evex_instruction();
10191 attributes.set_embedded_opmask_register_specifier(mask);
10192 if (merge) {
10193 attributes.reset_is_clear_context();
10194 }
10195 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10196 emit_int8((unsigned char)0xEA);
10197 emit_operand(dst, src, 0);
10198 }
10199
10200 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10201 assert(VM_Version::supports_evex(), "");
10202 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10203 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10204 attributes.set_is_evex_instruction();
10205 attributes.set_embedded_opmask_register_specifier(mask);
10206 if (merge) {
10207 attributes.reset_is_clear_context();
10208 }
10209 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10210 emit_int16(0x39, (0xC0 | encode));
10211 }
10212
10213 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10214 assert(VM_Version::supports_evex(), "");
10215 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10216 InstructionMark im(this);
10217 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10218 attributes.set_is_evex_instruction();
10219 attributes.set_embedded_opmask_register_specifier(mask);
10220 if (merge) {
10221 attributes.reset_is_clear_context();
10222 }
10223 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10224 emit_int8(0x39);
10225 emit_operand(dst, src, 0);
10226 }
10227
10228 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10229 assert(VM_Version::supports_evex(), "");
10230 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10231 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10232 attributes.set_is_evex_instruction();
10233 attributes.set_embedded_opmask_register_specifier(mask);
10234 if (merge) {
10235 attributes.reset_is_clear_context();
10236 }
10237 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10238 emit_int16(0x39, (0xC0 | encode));
10239 }
10240
10241 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10242 assert(VM_Version::supports_evex(), "");
10243 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10244 InstructionMark im(this);
10245 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10246 attributes.set_is_evex_instruction();
10247 attributes.set_embedded_opmask_register_specifier(mask);
10248 if (merge) {
10249 attributes.reset_is_clear_context();
10250 }
10251 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10252 emit_int8(0x39);
10253 emit_operand(dst, src, 0);
10254 }
10255
10256
10257 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10258 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10259 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10260 attributes.set_is_evex_instruction();
10261 attributes.set_embedded_opmask_register_specifier(mask);
10262 if (merge) {
10263 attributes.reset_is_clear_context();
10264 }
10265 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10266 emit_int16(0x3C, (0xC0 | encode));
10267 }
10268
10269 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10270 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10271 InstructionMark im(this);
10272 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10273 attributes.set_is_evex_instruction();
10274 attributes.set_embedded_opmask_register_specifier(mask);
10275 if (merge) {
10276 attributes.reset_is_clear_context();
10277 }
10278 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10279 emit_int8(0x3C);
10280 emit_operand(dst, src, 0);
10281 }
10282
10283 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10284 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10285 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10286 attributes.set_is_evex_instruction();
10287 attributes.set_embedded_opmask_register_specifier(mask);
10288 if (merge) {
10289 attributes.reset_is_clear_context();
10290 }
10291 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10292 emit_int16((unsigned char)0xEE, (0xC0 | encode));
10293 }
10294
10295 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10296 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
10297 InstructionMark im(this);
10298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10299 attributes.set_is_evex_instruction();
10300 attributes.set_embedded_opmask_register_specifier(mask);
10301 if (merge) {
10302 attributes.reset_is_clear_context();
10303 }
10304 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10305 emit_int8((unsigned char)0xEE);
10306 emit_operand(dst, src, 0);
10307 }
10308
10309 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10310 assert(VM_Version::supports_evex(), "");
10311 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10312 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10313 attributes.set_is_evex_instruction();
10314 attributes.set_embedded_opmask_register_specifier(mask);
10315 if (merge) {
10316 attributes.reset_is_clear_context();
10317 }
10318 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10319 emit_int16(0x3D, (0xC0 | encode));
10320 }
10321
10322 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10323 assert(VM_Version::supports_evex(), "");
10324 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10325 InstructionMark im(this);
10326 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10327 attributes.set_is_evex_instruction();
10328 attributes.set_embedded_opmask_register_specifier(mask);
10329 if (merge) {
10330 attributes.reset_is_clear_context();
10331 }
10332 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10333 emit_int8(0x3D);
10334 emit_operand(dst, src, 0);
10335 }
10336
10337 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10338 assert(VM_Version::supports_evex(), "");
10339 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10340 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10341 attributes.set_is_evex_instruction();
10342 attributes.set_embedded_opmask_register_specifier(mask);
10343 if (merge) {
10344 attributes.reset_is_clear_context();
10345 }
10346 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10347 emit_int16(0x3D, (0xC0 | encode));
10348 }
10349
10350 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10351 assert(VM_Version::supports_evex(), "");
10352 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10353 InstructionMark im(this);
10354 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10355 attributes.set_is_evex_instruction();
10356 attributes.set_embedded_opmask_register_specifier(mask);
10357 if (merge) {
10358 attributes.reset_is_clear_context();
10359 }
10360 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10361 emit_int8(0x3D);
10362 emit_operand(dst, src, 0);
10363 }
10364
10365 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10366 assert(VM_Version::supports_evex(), "requires EVEX support");
10367 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10368 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10369 attributes.set_is_evex_instruction();
10370 attributes.set_embedded_opmask_register_specifier(mask);
10371 if (merge) {
10372 attributes.reset_is_clear_context();
10373 }
10374 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10375 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10376 }
10377
10378 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10379 assert(VM_Version::supports_evex(), "requires EVEX support");
10380 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10381 assert(dst != xnoreg, "sanity");
10382 InstructionMark im(this);
10383 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10384 attributes.set_is_evex_instruction();
10385 attributes.set_embedded_opmask_register_specifier(mask);
10386 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10387 if (merge) {
10388 attributes.reset_is_clear_context();
10389 }
10390 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10391 emit_int8(0x25);
10392 emit_operand(dst, src3, 1);
10393 emit_int8(imm8);
10394 }
10395
10396 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10397 assert(VM_Version::supports_evex(), "requires EVEX support");
10398 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10399 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10400 attributes.set_is_evex_instruction();
10401 attributes.set_embedded_opmask_register_specifier(mask);
10402 if (merge) {
10403 attributes.reset_is_clear_context();
10404 }
10405 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10406 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10407 }
10408
10409 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10410 assert(VM_Version::supports_evex(), "requires EVEX support");
10411 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10412 assert(dst != xnoreg, "sanity");
10413 InstructionMark im(this);
10414 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10415 attributes.set_is_evex_instruction();
10416 attributes.set_embedded_opmask_register_specifier(mask);
10417 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10418 if (merge) {
10419 attributes.reset_is_clear_context();
10420 }
10421 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10422 emit_int8(0x25);
10423 emit_operand(dst, src3, 1);
10424 emit_int8(imm8);
10425 }
10426
10427 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10428 assert(VM_Version::supports_gfni(), "");
10429 assert(VM_Version::supports_sse(), "");
10430 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10431 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10432 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10433 }
10434
10435 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10436 assert(VM_Version::supports_gfni(), "requires GFNI support");
10437 assert(VM_Version::supports_sse(), "");
10438 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10439 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10440 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10441 }
10442
10443 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10444 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10445 assert(UseAVX >= 2, "");
10446 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10447 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10448 emit_int16(0x58, (0xC0 | encode));
10449 }
10450
10451 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10452 assert(VM_Version::supports_avx2(), "");
10453 assert(dst != xnoreg, "sanity");
10454 InstructionMark im(this);
10455 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10456 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10457 // swap src<->dst for encoding
10458 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10459 emit_int8(0x58);
10460 emit_operand(dst, src, 0);
10461 }
10462
10463 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10464 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10465 assert(VM_Version::supports_avx2(), "");
10466 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10467 attributes.set_rex_vex_w_reverted();
10468 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10469 emit_int16(0x59, (0xC0 | encode));
10470 }
10471
10472 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10473 assert(VM_Version::supports_avx2(), "");
10474 assert(dst != xnoreg, "sanity");
10475 InstructionMark im(this);
10476 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10477 attributes.set_rex_vex_w_reverted();
10478 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10479 // swap src<->dst for encoding
10480 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10481 emit_int8(0x59);
10482 emit_operand(dst, src, 0);
10483 }
10484
10485 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10486 assert(vector_len != Assembler::AVX_128bit, "");
10487 assert(VM_Version::supports_evex(), "");
10488 assert(dst != xnoreg, "sanity");
10489 InstructionMark im(this);
10490 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10491 attributes.set_rex_vex_w_reverted();
10492 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10493 // swap src<->dst for encoding
10494 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10495 emit_int8(0x5A);
10496 emit_operand(dst, src, 0);
10497 }
10498
10499 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10500 assert(vector_len != Assembler::AVX_128bit, "");
10501 assert(VM_Version::supports_avx512dq(), "");
10502 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10503 attributes.set_rex_vex_w_reverted();
10504 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10505 emit_int16(0x5A, (0xC0 | encode));
10506 }
10507
10508 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10509 assert(vector_len != Assembler::AVX_128bit, "");
10510 assert(VM_Version::supports_avx512dq(), "");
10511 assert(dst != xnoreg, "sanity");
10512 InstructionMark im(this);
10513 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10514 attributes.set_rex_vex_w_reverted();
10515 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10516 // swap src<->dst for encoding
10517 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10518 emit_int8(0x5A);
10519 emit_operand(dst, src, 0);
10520 }
10521
10522 // scalar single/double precision replicate
10523
10524 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10525 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10526 assert(VM_Version::supports_avx2(), "");
10527 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10528 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10529 emit_int16(0x18, (0xC0 | encode));
10530 }
10531
10532 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10533 assert(VM_Version::supports_avx(), "");
10534 assert(dst != xnoreg, "sanity");
10535 InstructionMark im(this);
10536 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10537 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10538 // swap src<->dst for encoding
10539 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10540 emit_int8(0x18);
10541 emit_operand(dst, src, 0);
10542 }
10543
10544 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10545 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10546 assert(VM_Version::supports_avx2(), "");
10547 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10548 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10549 attributes.set_rex_vex_w_reverted();
10550 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10551 emit_int16(0x19, (0xC0 | encode));
10552 }
10553
10554 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10555 assert(VM_Version::supports_avx(), "");
10556 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10557 assert(dst != xnoreg, "sanity");
10558 InstructionMark im(this);
10559 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10560 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10561 attributes.set_rex_vex_w_reverted();
10562 // swap src<->dst for encoding
10563 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10564 emit_int8(0x19);
10565 emit_operand(dst, src, 0);
10566 }
10567
10568 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10569 assert(VM_Version::supports_avx(), "");
10570 assert(vector_len == AVX_256bit, "");
10571 assert(dst != xnoreg, "sanity");
10572 InstructionMark im(this);
10573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10574 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10575 // swap src<->dst for encoding
10576 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10577 emit_int8(0x1A);
10578 emit_operand(dst, src, 0);
10579 }
10580
10581 // gpr source broadcast forms
10582
10583 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10584 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10585 assert(VM_Version::supports_avx512bw(), "");
10586 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10587 attributes.set_is_evex_instruction();
10588 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10589 emit_int16(0x7A, (0xC0 | encode));
10590 }
10591
10592 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10593 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10594 assert(VM_Version::supports_avx512bw(), "");
10595 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10596 attributes.set_is_evex_instruction();
10597 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10598 emit_int16(0x7B, (0xC0 | encode));
10599 }
10600
10601 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10602 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10603 assert(VM_Version::supports_evex(), "");
10604 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10605 attributes.set_is_evex_instruction();
10606 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10607 emit_int16(0x7C, (0xC0 | encode));
10608 }
10609
10610 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10611 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10612 assert(VM_Version::supports_evex(), "");
10613 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10614 attributes.set_is_evex_instruction();
10615 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10616 emit_int16(0x7C, (0xC0 | encode));
10617 }
10618
10619 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10620 assert(VM_Version::supports_avx2(), "");
10621 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10622 assert(dst != xnoreg, "sanity");
10623 assert(src.isxmmindex(),"expected to be xmm index");
10624 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10625 InstructionMark im(this);
10626 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10627 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10628 emit_int8((unsigned char)0x90);
10629 emit_operand(dst, src, 0);
10630 }
10631
10632 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10633 assert(VM_Version::supports_avx2(), "");
10634 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10635 assert(dst != xnoreg, "sanity");
10636 assert(src.isxmmindex(),"expected to be xmm index");
10637 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10638 InstructionMark im(this);
10639 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10640 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10641 emit_int8((unsigned char)0x90);
10642 emit_operand(dst, src, 0);
10643 }
10644
10645 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10646 assert(VM_Version::supports_avx2(), "");
10647 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10648 assert(dst != xnoreg, "sanity");
10649 assert(src.isxmmindex(),"expected to be xmm index");
10650 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10651 InstructionMark im(this);
10652 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10653 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10654 emit_int8((unsigned char)0x92);
10655 emit_operand(dst, src, 0);
10656 }
10657
10658 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10659 assert(VM_Version::supports_avx2(), "");
10660 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10661 assert(dst != xnoreg, "sanity");
10662 assert(src.isxmmindex(),"expected to be xmm index");
10663 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10664 InstructionMark im(this);
10665 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10666 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10667 emit_int8((unsigned char)0x92);
10668 emit_operand(dst, src, 0);
10669 }
10670 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10671 assert(VM_Version::supports_evex(), "");
10672 assert(dst != xnoreg, "sanity");
10673 assert(src.isxmmindex(),"expected to be xmm index");
10674 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10675 assert(mask != k0, "instruction will #UD if mask is in k0");
10676 InstructionMark im(this);
10677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10678 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10679 attributes.reset_is_clear_context();
10680 attributes.set_embedded_opmask_register_specifier(mask);
10681 attributes.set_is_evex_instruction();
10682 // swap src<->dst for encoding
10683 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10684 emit_int8((unsigned char)0x90);
10685 emit_operand(dst, src, 0);
10686 }
10687
10688 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10689 assert(VM_Version::supports_evex(), "");
10690 assert(dst != xnoreg, "sanity");
10691 assert(src.isxmmindex(),"expected to be xmm index");
10692 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10693 assert(mask != k0, "instruction will #UD if mask is in k0");
10694 InstructionMark im(this);
10695 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10696 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10697 attributes.reset_is_clear_context();
10698 attributes.set_embedded_opmask_register_specifier(mask);
10699 attributes.set_is_evex_instruction();
10700 // swap src<->dst for encoding
10701 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10702 emit_int8((unsigned char)0x90);
10703 emit_operand(dst, src, 0);
10704 }
10705
10706 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10707 assert(VM_Version::supports_evex(), "");
10708 assert(dst != xnoreg, "sanity");
10709 assert(src.isxmmindex(),"expected to be xmm index");
10710 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10711 assert(mask != k0, "instruction will #UD if mask is in k0");
10712 InstructionMark im(this);
10713 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10714 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10715 attributes.reset_is_clear_context();
10716 attributes.set_embedded_opmask_register_specifier(mask);
10717 attributes.set_is_evex_instruction();
10718 // swap src<->dst for encoding
10719 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10720 emit_int8((unsigned char)0x92);
10721 emit_operand(dst, src, 0);
10722 }
10723
10724 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10725 assert(VM_Version::supports_evex(), "");
10726 assert(dst != xnoreg, "sanity");
10727 assert(src.isxmmindex(),"expected to be xmm index");
10728 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10729 assert(mask != k0, "instruction will #UD if mask is in k0");
10730 InstructionMark im(this);
10731 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10732 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10733 attributes.reset_is_clear_context();
10734 attributes.set_embedded_opmask_register_specifier(mask);
10735 attributes.set_is_evex_instruction();
10736 // swap src<->dst for encoding
10737 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10738 emit_int8((unsigned char)0x92);
10739 emit_operand(dst, src, 0);
10740 }
10741
10742 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10743 assert(VM_Version::supports_evex(), "");
10744 assert(mask != k0, "instruction will #UD if mask is in k0");
10745 InstructionMark im(this);
10746 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10747 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10748 attributes.reset_is_clear_context();
10749 attributes.set_embedded_opmask_register_specifier(mask);
10750 attributes.set_is_evex_instruction();
10751 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10752 emit_int8((unsigned char)0xA0);
10753 emit_operand(src, dst, 0);
10754 }
10755
10756 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10757 assert(VM_Version::supports_evex(), "");
10758 assert(mask != k0, "instruction will #UD if mask is in k0");
10759 InstructionMark im(this);
10760 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10761 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10762 attributes.reset_is_clear_context();
10763 attributes.set_embedded_opmask_register_specifier(mask);
10764 attributes.set_is_evex_instruction();
10765 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10766 emit_int8((unsigned char)0xA0);
10767 emit_operand(src, dst, 0);
10768 }
10769
10770 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10771 assert(VM_Version::supports_evex(), "");
10772 assert(mask != k0, "instruction will #UD if mask is in k0");
10773 InstructionMark im(this);
10774 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10775 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10776 attributes.reset_is_clear_context();
10777 attributes.set_embedded_opmask_register_specifier(mask);
10778 attributes.set_is_evex_instruction();
10779 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10780 emit_int8((unsigned char)0xA2);
10781 emit_operand(src, dst, 0);
10782 }
10783
10784 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10785 assert(VM_Version::supports_evex(), "");
10786 assert(mask != k0, "instruction will #UD if mask is in k0");
10787 InstructionMark im(this);
10788 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10789 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10790 attributes.reset_is_clear_context();
10791 attributes.set_embedded_opmask_register_specifier(mask);
10792 attributes.set_is_evex_instruction();
10793 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10794 emit_int8((unsigned char)0xA2);
10795 emit_operand(src, dst, 0);
10796 }
10797 // Carry-Less Multiplication Quadword
10798 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10799 assert(VM_Version::supports_clmul(), "");
10800 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10801 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10802 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10803 }
10804
10805 // Carry-Less Multiplication Quadword
10806 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10807 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10808 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10809 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10810 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10811 }
10812
10813 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10814 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10815 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10816 attributes.set_is_evex_instruction();
10817 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10818 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10819 }
10820
10821 void Assembler::vzeroupper_uncached() {
10822 if (VM_Version::supports_vzeroupper()) {
10823 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10824 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10825 emit_int8(0x77);
10826 }
10827 }
10828
10829 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10830 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10831 assert(VM_Version::supports_evex(), "");
10832 assert(VM_Version::supports_avx512dq(), "");
10833 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10834 attributes.set_is_evex_instruction();
10835 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10836 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10837 }
10838
10839 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10840 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10841 assert(VM_Version::supports_evex(), "");
10842 assert(VM_Version::supports_avx512dq(), "");
10843 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10844 attributes.set_is_evex_instruction();
10845 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10846 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10847 }
10848
10849 void Assembler::fld_x(Address adr) {
10850 InstructionMark im(this);
10851 emit_int8((unsigned char)0xDB);
10852 emit_operand32(rbp, adr, 0);
10853 }
10854
10855 void Assembler::fstp_x(Address adr) {
10856 InstructionMark im(this);
10857 emit_int8((unsigned char)0xDB);
10858 emit_operand32(rdi, adr, 0);
10859 }
10860
10861 void Assembler::emit_operand32(Register reg, Address adr, int post_addr_length) {
10862 assert(reg->encoding() < 8, "no extended registers");
10863 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10864 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec, post_addr_length);
10865 }
10866
10867 #ifndef _LP64
10868 // 32bit only pieces of the assembler
10869
10870 void Assembler::emms() {
10871 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10872 emit_int16(0x0F, 0x77);
10873 }
10874
10875 void Assembler::vzeroupper() {
10876 vzeroupper_uncached();
10877 }
10878
10879 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10880 // NO PREFIX AS NEVER 64BIT
10881 InstructionMark im(this);
10882 emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10883 emit_data(imm32, rspec, 0);
10884 }
10885
10886 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10887 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10888 InstructionMark im(this);
10889 emit_int8((unsigned char)0x81);
10890 emit_operand(rdi, src1, 4);
10891 emit_data(imm32, rspec, 0);
10892 }
10893
10894 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10895 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10896 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
10897 void Assembler::cmpxchg8(Address adr) {
10898 InstructionMark im(this);
10899 emit_int16(0x0F, (unsigned char)0xC7);
10900 emit_operand(rcx, adr, 0);
10901 }
10902
10903 void Assembler::decl(Register dst) {
10904 // Don't use it directly. Use MacroAssembler::decrementl() instead.
10905 emit_int8(0x48 | dst->encoding());
10906 }
10907
10908 // 64bit doesn't use the x87
10909
10910 void Assembler::emit_farith(int b1, int b2, int i) {
10911 assert(isByte(b1) && isByte(b2), "wrong opcode");
10912 assert(0 <= i && i < 8, "illegal stack offset");
10913 emit_int16(b1, b2 + i);
10914 }
10915
10916 void Assembler::fabs() {
10917 emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10918 }
10919
10920 void Assembler::fadd(int i) {
10921 emit_farith(0xD8, 0xC0, i);
10922 }
10923
10924 void Assembler::fadd_d(Address src) {
10925 InstructionMark im(this);
10926 emit_int8((unsigned char)0xDC);
10927 emit_operand32(rax, src, 0);
10928 }
10929
10930 void Assembler::fadd_s(Address src) {
10931 InstructionMark im(this);
10932 emit_int8((unsigned char)0xD8);
10933 emit_operand32(rax, src, 0);
10934 }
10935
10936 void Assembler::fadda(int i) {
10937 emit_farith(0xDC, 0xC0, i);
10938 }
10939
10940 void Assembler::faddp(int i) {
10941 emit_farith(0xDE, 0xC0, i);
10942 }
10943
10944 void Assembler::fchs() {
10945 emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10946 }
10947
10948 void Assembler::fcom(int i) {
10949 emit_farith(0xD8, 0xD0, i);
10950 }
10951
10952 void Assembler::fcomp(int i) {
10953 emit_farith(0xD8, 0xD8, i);
10954 }
10955
10956 void Assembler::fcomp_d(Address src) {
10957 InstructionMark im(this);
10958 emit_int8((unsigned char)0xDC);
10959 emit_operand32(rbx, src, 0);
10960 }
10961
10962 void Assembler::fcomp_s(Address src) {
10963 InstructionMark im(this);
10964 emit_int8((unsigned char)0xD8);
10965 emit_operand32(rbx, src, 0);
10966 }
10967
10968 void Assembler::fcompp() {
10969 emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10970 }
10971
10972 void Assembler::fcos() {
10973 emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10974 }
10975
10976 void Assembler::fdecstp() {
10977 emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10978 }
10979
10980 void Assembler::fdiv(int i) {
10981 emit_farith(0xD8, 0xF0, i);
10982 }
10983
10984 void Assembler::fdiv_d(Address src) {
10985 InstructionMark im(this);
10986 emit_int8((unsigned char)0xDC);
10987 emit_operand32(rsi, src, 0);
10988 }
10989
10990 void Assembler::fdiv_s(Address src) {
10991 InstructionMark im(this);
10992 emit_int8((unsigned char)0xD8);
10993 emit_operand32(rsi, src, 0);
10994 }
10995
10996 void Assembler::fdiva(int i) {
10997 emit_farith(0xDC, 0xF8, i);
10998 }
10999
11000 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
11001 // is erroneous for some of the floating-point instructions below.
11002
11003 void Assembler::fdivp(int i) {
11004 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
11005 }
11006
11007 void Assembler::fdivr(int i) {
11008 emit_farith(0xD8, 0xF8, i);
11009 }
11010
11011 void Assembler::fdivr_d(Address src) {
11012 InstructionMark im(this);
11013 emit_int8((unsigned char)0xDC);
11014 emit_operand32(rdi, src, 0);
11015 }
11016
11017 void Assembler::fdivr_s(Address src) {
11018 InstructionMark im(this);
11019 emit_int8((unsigned char)0xD8);
11020 emit_operand32(rdi, src, 0);
11021 }
11022
11023 void Assembler::fdivra(int i) {
11024 emit_farith(0xDC, 0xF0, i);
11025 }
11026
11027 void Assembler::fdivrp(int i) {
11028 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
11029 }
11030
11031 void Assembler::ffree(int i) {
11032 emit_farith(0xDD, 0xC0, i);
11033 }
11034
11035 void Assembler::fild_d(Address adr) {
11036 InstructionMark im(this);
11037 emit_int8((unsigned char)0xDF);
11038 emit_operand32(rbp, adr, 0);
11039 }
11040
11041 void Assembler::fild_s(Address adr) {
11042 InstructionMark im(this);
11043 emit_int8((unsigned char)0xDB);
11044 emit_operand32(rax, adr, 0);
11045 }
11046
11047 void Assembler::fincstp() {
11048 emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
11049 }
11050
11051 void Assembler::finit() {
11052 emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
11053 }
11054
11055 void Assembler::fist_s(Address adr) {
11056 InstructionMark im(this);
11057 emit_int8((unsigned char)0xDB);
11058 emit_operand32(rdx, adr, 0);
11059 }
11060
11061 void Assembler::fistp_d(Address adr) {
11062 InstructionMark im(this);
11063 emit_int8((unsigned char)0xDF);
11064 emit_operand32(rdi, adr, 0);
11065 }
11066
11067 void Assembler::fistp_s(Address adr) {
11068 InstructionMark im(this);
11069 emit_int8((unsigned char)0xDB);
11070 emit_operand32(rbx, adr, 0);
11071 }
11072
11073 void Assembler::fld1() {
11074 emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
11075 }
11076
11077 void Assembler::fld_d(Address adr) {
11078 InstructionMark im(this);
11079 emit_int8((unsigned char)0xDD);
11080 emit_operand32(rax, adr, 0);
11081 }
11082
11083 void Assembler::fld_s(Address adr) {
11084 InstructionMark im(this);
11085 emit_int8((unsigned char)0xD9);
11086 emit_operand32(rax, adr, 0);
11087 }
11088
11089
11090 void Assembler::fld_s(int index) {
11091 emit_farith(0xD9, 0xC0, index);
11092 }
11093
11094 void Assembler::fldcw(Address src) {
11095 InstructionMark im(this);
11096 emit_int8((unsigned char)0xD9);
11097 emit_operand32(rbp, src, 0);
11098 }
11099
11100 void Assembler::fldenv(Address src) {
11101 InstructionMark im(this);
11102 emit_int8((unsigned char)0xD9);
11103 emit_operand32(rsp, src, 0);
11104 }
11105
11106 void Assembler::fldlg2() {
11107 emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
11108 }
11109
11110 void Assembler::fldln2() {
11111 emit_int16((unsigned char)0xD9, (unsigned char)0xED);
11112 }
11113
11114 void Assembler::fldz() {
11115 emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
11116 }
11117
11118 void Assembler::flog() {
11119 fldln2();
11120 fxch();
11121 fyl2x();
11122 }
11123
11124 void Assembler::flog10() {
11125 fldlg2();
11126 fxch();
11127 fyl2x();
11128 }
11129
11130 void Assembler::fmul(int i) {
11131 emit_farith(0xD8, 0xC8, i);
11132 }
11133
11134 void Assembler::fmul_d(Address src) {
11135 InstructionMark im(this);
11136 emit_int8((unsigned char)0xDC);
11137 emit_operand32(rcx, src, 0);
11138 }
11139
11140 void Assembler::fmul_s(Address src) {
11141 InstructionMark im(this);
11142 emit_int8((unsigned char)0xD8);
11143 emit_operand32(rcx, src, 0);
11144 }
11145
11146 void Assembler::fmula(int i) {
11147 emit_farith(0xDC, 0xC8, i);
11148 }
11149
11150 void Assembler::fmulp(int i) {
11151 emit_farith(0xDE, 0xC8, i);
11152 }
11153
11154 void Assembler::fnsave(Address dst) {
11155 InstructionMark im(this);
11156 emit_int8((unsigned char)0xDD);
11157 emit_operand32(rsi, dst, 0);
11158 }
11159
11160 void Assembler::fnstcw(Address src) {
11161 InstructionMark im(this);
11162 emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
11163 emit_operand32(rdi, src, 0);
11164 }
11165
11166 void Assembler::fnstsw_ax() {
11167 emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
11168 }
11169
11170 void Assembler::fprem() {
11171 emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
11172 }
11173
11174 void Assembler::fprem1() {
11175 emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
11176 }
11177
11178 void Assembler::frstor(Address src) {
11179 InstructionMark im(this);
11180 emit_int8((unsigned char)0xDD);
11181 emit_operand32(rsp, src, 0);
11182 }
11183
11184 void Assembler::fsin() {
11185 emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
11186 }
11187
11188 void Assembler::fsqrt() {
11189 emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
11190 }
11191
11192 void Assembler::fst_d(Address adr) {
11193 InstructionMark im(this);
11194 emit_int8((unsigned char)0xDD);
11195 emit_operand32(rdx, adr, 0);
11196 }
11197
11198 void Assembler::fst_s(Address adr) {
11199 InstructionMark im(this);
11200 emit_int8((unsigned char)0xD9);
11201 emit_operand32(rdx, adr, 0);
11202 }
11203
11204 void Assembler::fstp_d(Address adr) {
11205 InstructionMark im(this);
11206 emit_int8((unsigned char)0xDD);
11207 emit_operand32(rbx, adr, 0);
11208 }
11209
11210 void Assembler::fstp_d(int index) {
11211 emit_farith(0xDD, 0xD8, index);
11212 }
11213
11214 void Assembler::fstp_s(Address adr) {
11215 InstructionMark im(this);
11216 emit_int8((unsigned char)0xD9);
11217 emit_operand32(rbx, adr, 0);
11218 }
11219
11220 void Assembler::fsub(int i) {
11221 emit_farith(0xD8, 0xE0, i);
11222 }
11223
11224 void Assembler::fsub_d(Address src) {
11225 InstructionMark im(this);
11226 emit_int8((unsigned char)0xDC);
11227 emit_operand32(rsp, src, 0);
11228 }
11229
11230 void Assembler::fsub_s(Address src) {
11231 InstructionMark im(this);
11232 emit_int8((unsigned char)0xD8);
11233 emit_operand32(rsp, src, 0);
11234 }
11235
11236 void Assembler::fsuba(int i) {
11237 emit_farith(0xDC, 0xE8, i);
11238 }
11239
11240 void Assembler::fsubp(int i) {
11241 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
11242 }
11243
11244 void Assembler::fsubr(int i) {
11245 emit_farith(0xD8, 0xE8, i);
11246 }
11247
11248 void Assembler::fsubr_d(Address src) {
11249 InstructionMark im(this);
11250 emit_int8((unsigned char)0xDC);
11251 emit_operand32(rbp, src, 0);
11252 }
11253
11254 void Assembler::fsubr_s(Address src) {
11255 InstructionMark im(this);
11256 emit_int8((unsigned char)0xD8);
11257 emit_operand32(rbp, src, 0);
11258 }
11259
11260 void Assembler::fsubra(int i) {
11261 emit_farith(0xDC, 0xE0, i);
11262 }
11263
11264 void Assembler::fsubrp(int i) {
11265 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
11266 }
11267
11268 void Assembler::ftan() {
11269 emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
11270 }
11271
11272 void Assembler::ftst() {
11273 emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
11274 }
11275
11276 void Assembler::fucomi(int i) {
11277 // make sure the instruction is supported (introduced for P6, together with cmov)
11278 guarantee(VM_Version::supports_cmov(), "illegal instruction");
11279 emit_farith(0xDB, 0xE8, i);
11280 }
11281
11282 void Assembler::fucomip(int i) {
11283 // make sure the instruction is supported (introduced for P6, together with cmov)
11284 guarantee(VM_Version::supports_cmov(), "illegal instruction");
11285 emit_farith(0xDF, 0xE8, i);
11286 }
11287
11288 void Assembler::fwait() {
11289 emit_int8((unsigned char)0x9B);
11290 }
11291
11292 void Assembler::fxch(int i) {
11293 emit_farith(0xD9, 0xC8, i);
11294 }
11295
11296 void Assembler::fyl2x() {
11297 emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
11298 }
11299
11300 void Assembler::frndint() {
11301 emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
11302 }
11303
11304 void Assembler::f2xm1() {
11305 emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11306 }
11307
11308 void Assembler::fldl2e() {
11309 emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11310 }
11311 #endif // !_LP64
11312
11313 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11314 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11315 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11316 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
11317
11318 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11319 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11320 if (pre > 0) {
11321 emit_int8(simd_pre[pre]);
11322 }
11323 if (rex_w) {
11324 prefixq(adr, xreg);
11325 } else {
11326 prefix(adr, xreg);
11327 }
11328 if (opc > 0) {
11329 emit_int8(0x0F);
11330 int opc2 = simd_opc[opc];
11331 if (opc2 > 0) {
11332 emit_int8(opc2);
11333 }
11334 }
11335 }
11336
11337 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11338 if (pre > 0) {
11339 emit_int8(simd_pre[pre]);
11340 }
11341 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11342 if (opc > 0) {
11343 emit_int8(0x0F);
11344 int opc2 = simd_opc[opc];
11345 if (opc2 > 0) {
11346 emit_int8(opc2);
11347 }
11348 }
11349 return encode;
11350 }
11351
11352
11353 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11354 int vector_len = _attributes->get_vector_len();
11355 bool vex_w = _attributes->is_rex_vex_w();
11356 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11357 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11358 byte1 = (~byte1) & 0xE0;
11359 byte1 |= opc;
11360
11361 int byte2 = ((~nds_enc) & 0xf) << 3;
11362 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11363
11364 emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11365 } else {
11366 int byte1 = vex_r ? VEX_R : 0;
11367 byte1 = (~byte1) & 0x80;
11368 byte1 |= ((~nds_enc) & 0xf) << 3;
11369 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11370 emit_int16((unsigned char)VEX_2bytes, byte1);
11371 }
11372 }
11373
11374 // This is a 4 byte encoding
11375 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11376 // EVEX 0x62 prefix
11377 // byte1 = EVEX_4bytes;
11378
11379 bool vex_w = _attributes->is_rex_vex_w();
11380 int evex_encoding = (vex_w ? VEX_W : 0);
11381 // EVEX.b is not currently used for broadcast of single element or data rounding modes
11382 _attributes->set_evex_encoding(evex_encoding);
11383
11384 // P0: byte 2, initialized to RXBR`00mm
11385 // instead of not'd
11386 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11387 byte2 = (~byte2) & 0xF0;
11388 // confine opc opcode extensions in mm bits to lower two bits
11389 // of form {0F, 0F_38, 0F_3A}
11390 byte2 |= opc;
11391
11392 // P1: byte 3 as Wvvvv1pp
11393 int byte3 = ((~nds_enc) & 0xf) << 3;
11394 // p[10] is always 1
11395 byte3 |= EVEX_F;
11396 byte3 |= (vex_w & 1) << 7;
11397 // confine pre opcode extensions in pp bits to lower two bits
11398 // of form {66, F3, F2}
11399 byte3 |= pre;
11400
11401 // P2: byte 4 as zL'Lbv'aaa
11402 // kregs are implemented in the low 3 bits as aaa
11403 int byte4 = (_attributes->is_no_reg_mask()) ?
11404 0 :
11405 _attributes->get_embedded_opmask_register_specifier();
11406 // EVEX.v` for extending EVEX.vvvv or VIDX
11407 byte4 |= (evex_v ? 0: EVEX_V);
11408 // third EXEC.b for broadcast actions
11409 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11410 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11411 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11412 // last is EVEX.z for zero/merge actions
11413 if (_attributes->is_no_reg_mask() == false &&
11414 _attributes->get_embedded_opmask_register_specifier() != 0) {
11415 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11416 }
11417
11418 emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11419 }
11420
11421 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11422 bool vex_r = (xreg_enc & 8) == 8;
11423 bool vex_b = adr.base_needs_rex();
11424 bool vex_x;
11425 if (adr.isxmmindex()) {
11426 vex_x = adr.xmmindex_needs_rex();
11427 } else {
11428 vex_x = adr.index_needs_rex();
11429 }
11430 set_attributes(attributes);
11431 attributes->set_current_assembler(this);
11432
11433 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11434 // is allowed in legacy mode and has resources which will fit in it.
11435 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11436 if (!attributes->is_legacy_mode()) {
11437 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11438 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11439 attributes->set_is_legacy_mode();
11440 }
11441 }
11442 }
11443
11444 if (UseAVX > 2) {
11445 assert(((!attributes->uses_vl()) ||
11446 (attributes->get_vector_len() == AVX_512bit) ||
11447 (!_legacy_mode_vl) ||
11448 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11449 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11450 }
11451
11452 clear_managed();
11453 if (UseAVX > 2 && !attributes->is_legacy_mode())
11454 {
11455 bool evex_r = (xreg_enc >= 16);
11456 bool evex_v;
11457 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11458 if (adr.isxmmindex()) {
11459 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11460 } else {
11461 evex_v = (nds_enc >= 16);
11462 }
11463 attributes->set_is_evex_instruction();
11464 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11465 } else {
11466 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11467 attributes->set_rex_vex_w(false);
11468 }
11469 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11470 }
11471 }
11472
11473 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11474 bool vex_r = (dst_enc & 8) == 8;
11475 bool vex_b = (src_enc & 8) == 8;
11476 bool vex_x = false;
11477 set_attributes(attributes);
11478 attributes->set_current_assembler(this);
11479
11480 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11481 // is allowed in legacy mode and has resources which will fit in it.
11482 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11483 if (!attributes->is_legacy_mode()) {
11484 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11485 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11486 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11487 attributes->set_is_legacy_mode();
11488 }
11489 }
11490 }
11491
11492 if (UseAVX > 2) {
11493 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11494 // Instruction with uses_vl true are vector instructions
11495 // All the vector instructions with AVX_512bit length can have legacy_mode as false
11496 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11497 // Rest all should have legacy_mode set as true
11498 assert(((!attributes->uses_vl()) ||
11499 (attributes->get_vector_len() == AVX_512bit) ||
11500 (!_legacy_mode_vl) ||
11501 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11502 // Instruction with legacy_mode true should have dst, nds and src < 15
11503 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11504 }
11505
11506 clear_managed();
11507 if (UseAVX > 2 && !attributes->is_legacy_mode())
11508 {
11509 bool evex_r = (dst_enc >= 16);
11510 bool evex_v = (nds_enc >= 16);
11511 // can use vex_x as bank extender on rm encoding
11512 vex_x = (src_enc >= 16);
11513 attributes->set_is_evex_instruction();
11514 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11515 } else {
11516 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11517 attributes->set_rex_vex_w(false);
11518 }
11519 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11520 }
11521
11522 // return modrm byte components for operands
11523 return (((dst_enc & 7) << 3) | (src_enc & 7));
11524 }
11525
11526
11527 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11528 VexOpcode opc, InstructionAttr *attributes) {
11529 if (UseAVX > 0) {
11530 int xreg_enc = xreg->encoding();
11531 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11532 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11533 } else {
11534 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11535 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11536 }
11537 }
11538
11539 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11540 VexOpcode opc, InstructionAttr *attributes) {
11541 int dst_enc = dst->encoding();
11542 int src_enc = src->encoding();
11543 if (UseAVX > 0) {
11544 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11545 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11546 } else {
11547 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11548 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11549 }
11550 }
11551
11552 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11553 assert(VM_Version::supports_avx(), "");
11554 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11555 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11556 emit_int16(0x5F, (0xC0 | encode));
11557 }
11558
11559 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11560 assert(VM_Version::supports_avx(), "");
11561 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11562 attributes.set_rex_vex_w_reverted();
11563 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11564 emit_int16(0x5F, (0xC0 | encode));
11565 }
11566
11567 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11568 assert(VM_Version::supports_avx(), "");
11569 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11570 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11571 emit_int16(0x5D, (0xC0 | encode));
11572 }
11573
11574 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11575 assert(VM_Version::supports_avx(), "");
11576 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11577 attributes.set_rex_vex_w_reverted();
11578 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11579 emit_int16(0x5D, (0xC0 | encode));
11580 }
11581
11582 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11583 assert(VM_Version::supports_avx(), "");
11584 assert(vector_len <= AVX_256bit, "");
11585 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11586 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11587 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11588 }
11589
11590 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11591 assert(VM_Version::supports_avx(), "");
11592 assert(vector_len <= AVX_256bit, "");
11593 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11594 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11595 int src2_enc = src2->encoding();
11596 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11597 }
11598
11599 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11600 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11601 assert(vector_len <= AVX_256bit, "");
11602 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11603 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11604 int src2_enc = src2->encoding();
11605 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11606 }
11607
11608 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11609 assert(VM_Version::supports_avx2(), "");
11610 assert(vector_len <= AVX_256bit, "");
11611 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11612 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11613 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11614 }
11615
11616 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11617 assert(VM_Version::supports_avx(), "");
11618 assert(vector_len <= AVX_256bit, "");
11619 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11620 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11621 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11622 }
11623
11624 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11625 ComparisonPredicateFP comparison, int vector_len) {
11626 assert(VM_Version::supports_evex(), "");
11627 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11628 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11629 attributes.set_is_evex_instruction();
11630 attributes.set_embedded_opmask_register_specifier(mask);
11631 attributes.reset_is_clear_context();
11632 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11633 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11634 }
11635
11636 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11637 ComparisonPredicateFP comparison, int vector_len) {
11638 assert(VM_Version::supports_evex(), "");
11639 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11640 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11641 attributes.set_is_evex_instruction();
11642 attributes.set_embedded_opmask_register_specifier(mask);
11643 attributes.reset_is_clear_context();
11644 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11645 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11646 }
11647
11648 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11649 assert(VM_Version::supports_sse4_1(), "");
11650 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11651 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11652 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11653 emit_int16(0x14, (0xC0 | encode));
11654 }
11655
11656 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11657 assert(VM_Version::supports_sse4_1(), "");
11658 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11659 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11660 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11661 emit_int16(0x15, (0xC0 | encode));
11662 }
11663
11664 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11665 assert(VM_Version::supports_sse4_1(), "");
11666 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11667 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11668 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11669 emit_int16(0x10, (0xC0 | encode));
11670 }
11671
11672 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11673 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11674 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11675 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11676 int src2_enc = src2->encoding();
11677 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11678 }
11679
11680 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11682 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11683 emit_int24(0x0C, (0xC0 | encode), imm8);
11684 }
11685
11686 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11687 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11688 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11689 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11690 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11691 emit_int16(0x64, (0xC0 | encode));
11692 }
11693
11694 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11695 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11696 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11697 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11698 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11699 emit_int16(0x65, (0xC0 | encode));
11700 }
11701
11702 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11703 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11704 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11705 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11706 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11707 emit_int16(0x66, (0xC0 | encode));
11708 }
11709
11710 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11711 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11712 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11713 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11714 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11715 emit_int16(0x37, (0xC0 | encode));
11716 }
11717
11718 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11719 int comparison, bool is_signed, int vector_len) {
11720 assert(VM_Version::supports_evex(), "");
11721 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11722 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11723 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11724 attributes.set_is_evex_instruction();
11725 attributes.set_embedded_opmask_register_specifier(mask);
11726 attributes.reset_is_clear_context();
11727 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11728 int opcode = is_signed ? 0x1F : 0x1E;
11729 emit_int24(opcode, (0xC0 | encode), comparison);
11730 }
11731
11732 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11733 int comparison, bool is_signed, int vector_len) {
11734 assert(VM_Version::supports_evex(), "");
11735 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11736 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11737 InstructionMark im(this);
11738 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11739 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11740 attributes.set_is_evex_instruction();
11741 attributes.set_embedded_opmask_register_specifier(mask);
11742 attributes.reset_is_clear_context();
11743 int dst_enc = kdst->encoding();
11744 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11745 int opcode = is_signed ? 0x1F : 0x1E;
11746 emit_int8((unsigned char)opcode);
11747 emit_operand(as_Register(dst_enc), src, 1);
11748 emit_int8((unsigned char)comparison);
11749 }
11750
11751 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11752 int comparison, bool is_signed, int vector_len) {
11753 assert(VM_Version::supports_evex(), "");
11754 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11755 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11756 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11757 attributes.set_is_evex_instruction();
11758 attributes.set_embedded_opmask_register_specifier(mask);
11759 attributes.reset_is_clear_context();
11760 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11761 int opcode = is_signed ? 0x1F : 0x1E;
11762 emit_int24(opcode, (0xC0 | encode), comparison);
11763 }
11764
11765 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11766 int comparison, bool is_signed, int vector_len) {
11767 assert(VM_Version::supports_evex(), "");
11768 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11769 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11770 InstructionMark im(this);
11771 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11772 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11773 attributes.set_is_evex_instruction();
11774 attributes.set_embedded_opmask_register_specifier(mask);
11775 attributes.reset_is_clear_context();
11776 int dst_enc = kdst->encoding();
11777 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11778 int opcode = is_signed ? 0x1F : 0x1E;
11779 emit_int8((unsigned char)opcode);
11780 emit_operand(as_Register(dst_enc), src, 1);
11781 emit_int8((unsigned char)comparison);
11782 }
11783
11784 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11785 int comparison, bool is_signed, int vector_len) {
11786 assert(VM_Version::supports_evex(), "");
11787 assert(VM_Version::supports_avx512bw(), "");
11788 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11789 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11790 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11791 attributes.set_is_evex_instruction();
11792 attributes.set_embedded_opmask_register_specifier(mask);
11793 attributes.reset_is_clear_context();
11794 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11795 int opcode = is_signed ? 0x3F : 0x3E;
11796 emit_int24(opcode, (0xC0 | encode), comparison);
11797 }
11798
11799 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11800 int comparison, bool is_signed, int vector_len) {
11801 assert(VM_Version::supports_evex(), "");
11802 assert(VM_Version::supports_avx512bw(), "");
11803 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11804 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11805 InstructionMark im(this);
11806 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11807 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11808 attributes.set_is_evex_instruction();
11809 attributes.set_embedded_opmask_register_specifier(mask);
11810 attributes.reset_is_clear_context();
11811 int dst_enc = kdst->encoding();
11812 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11813 int opcode = is_signed ? 0x3F : 0x3E;
11814 emit_int8((unsigned char)opcode);
11815 emit_operand(as_Register(dst_enc), src, 1);
11816 emit_int8((unsigned char)comparison);
11817 }
11818
11819 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11820 int comparison, bool is_signed, int vector_len) {
11821 assert(VM_Version::supports_evex(), "");
11822 assert(VM_Version::supports_avx512bw(), "");
11823 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11824 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11825 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11826 attributes.set_is_evex_instruction();
11827 attributes.set_embedded_opmask_register_specifier(mask);
11828 attributes.reset_is_clear_context();
11829 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11830 int opcode = is_signed ? 0x3F : 0x3E;
11831 emit_int24(opcode, (0xC0 | encode), comparison);
11832 }
11833
11834 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11835 int comparison, bool is_signed, int vector_len) {
11836 assert(VM_Version::supports_evex(), "");
11837 assert(VM_Version::supports_avx512bw(), "");
11838 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11839 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11840 InstructionMark im(this);
11841 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11842 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11843 attributes.set_is_evex_instruction();
11844 attributes.set_embedded_opmask_register_specifier(mask);
11845 attributes.reset_is_clear_context();
11846 int dst_enc = kdst->encoding();
11847 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11848 int opcode = is_signed ? 0x3F : 0x3E;
11849 emit_int8((unsigned char)opcode);
11850 emit_operand(as_Register(dst_enc), src, 1);
11851 emit_int8((unsigned char)comparison);
11852 }
11853
11854 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11855 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11857 attributes.set_is_evex_instruction();
11858 attributes.set_embedded_opmask_register_specifier(mask);
11859 if (merge) {
11860 attributes.reset_is_clear_context();
11861 }
11862 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11863 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11864 }
11865
11866 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11867 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11868 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11869 attributes.set_is_evex_instruction();
11870 attributes.set_embedded_opmask_register_specifier(mask);
11871 if (merge) {
11872 attributes.reset_is_clear_context();
11873 }
11874 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11875 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11876 }
11877
11878 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11879 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11880 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11881 attributes.set_is_evex_instruction();
11882 attributes.set_embedded_opmask_register_specifier(mask);
11883 if (merge) {
11884 attributes.reset_is_clear_context();
11885 }
11886 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11887 emit_int16(0x14, (0xC0 | encode));
11888 }
11889
11890 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11891 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11892 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11893 attributes.set_is_evex_instruction();
11894 attributes.set_embedded_opmask_register_specifier(mask);
11895 if (merge) {
11896 attributes.reset_is_clear_context();
11897 }
11898 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11899 emit_int16(0x14, (0xC0 | encode));
11900 }
11901
11902 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11903 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11904 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11905 attributes.set_is_evex_instruction();
11906 attributes.set_embedded_opmask_register_specifier(mask);
11907 if (merge) {
11908 attributes.reset_is_clear_context();
11909 }
11910 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11911 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11912 }
11913
11914 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11915 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11916 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11917 attributes.set_is_evex_instruction();
11918 attributes.set_embedded_opmask_register_specifier(mask);
11919 if (merge) {
11920 attributes.reset_is_clear_context();
11921 }
11922 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11923 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11924 }
11925
11926 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11927 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11928 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11929 attributes.set_is_evex_instruction();
11930 attributes.set_embedded_opmask_register_specifier(mask);
11931 if (merge) {
11932 attributes.reset_is_clear_context();
11933 }
11934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11935 emit_int16(0x15, (0xC0 | encode));
11936 }
11937
11938 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11939 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11940 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11941 attributes.set_is_evex_instruction();
11942 attributes.set_embedded_opmask_register_specifier(mask);
11943 if (merge) {
11944 attributes.reset_is_clear_context();
11945 }
11946 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11947 emit_int16(0x15, (0xC0 | encode));
11948 }
11949
11950 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11951 assert(VM_Version::supports_avx(), "");
11952 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11953 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11954 int mask_enc = mask->encoding();
11955 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11956 }
11957
11958 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11959 assert(VM_Version::supports_evex(), "");
11960 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11961 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11962 attributes.set_is_evex_instruction();
11963 attributes.set_embedded_opmask_register_specifier(mask);
11964 if (merge) {
11965 attributes.reset_is_clear_context();
11966 }
11967 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11968 emit_int16(0x65, (0xC0 | encode));
11969 }
11970
11971 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11972 assert(VM_Version::supports_evex(), "");
11973 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11974 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11975 attributes.set_is_evex_instruction();
11976 attributes.set_embedded_opmask_register_specifier(mask);
11977 if (merge) {
11978 attributes.reset_is_clear_context();
11979 }
11980 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11981 emit_int16(0x65, (0xC0 | encode));
11982 }
11983
11984 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11985 assert(VM_Version::supports_evex(), "");
11986 assert(VM_Version::supports_avx512bw(), "");
11987 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11988 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11989 attributes.set_is_evex_instruction();
11990 attributes.set_embedded_opmask_register_specifier(mask);
11991 if (merge) {
11992 attributes.reset_is_clear_context();
11993 }
11994 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11995 emit_int16(0x66, (0xC0 | encode));
11996 }
11997
11998 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11999 assert(VM_Version::supports_evex(), "");
12000 assert(VM_Version::supports_avx512bw(), "");
12001 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
12002 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
12003 attributes.set_is_evex_instruction();
12004 attributes.set_embedded_opmask_register_specifier(mask);
12005 if (merge) {
12006 attributes.reset_is_clear_context();
12007 }
12008 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12009 emit_int16(0x66, (0xC0 | encode));
12010 }
12011
12012 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12013 assert(VM_Version::supports_evex(), "");
12014 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
12015 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12016 attributes.set_is_evex_instruction();
12017 attributes.set_embedded_opmask_register_specifier(mask);
12018 if (merge) {
12019 attributes.reset_is_clear_context();
12020 }
12021 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12022 emit_int16(0x64, (0xC0 | encode));
12023 }
12024
12025 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
12026 assert(VM_Version::supports_evex(), "");
12027 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
12028 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12029 attributes.set_is_evex_instruction();
12030 attributes.set_embedded_opmask_register_specifier(mask);
12031 if (merge) {
12032 attributes.reset_is_clear_context();
12033 }
12034 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12035 emit_int16(0x64, (0xC0 | encode));
12036 }
12037
12038 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
12039 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12040 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12041 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12042 emit_int16((unsigned char)0xF5, (0xC0 | encode));
12043 }
12044
12045 void Assembler::pextl(Register dst, Register src1, Register src2) {
12046 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12047 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12048 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12049 emit_int16((unsigned char)0xF5, (0xC0 | encode));
12050 }
12051
12052 void Assembler::pdepl(Register dst, Register src1, Register src2) {
12053 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12054 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12055 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12056 emit_int16((unsigned char)0xF5, (0xC0 | encode));
12057 }
12058
12059 void Assembler::pextq(Register dst, Register src1, Register src2) {
12060 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12061 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12062 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12063 emit_int16((unsigned char)0xF5, (0xC0 | encode));
12064 }
12065
12066 void Assembler::pdepq(Register dst, Register src1, Register src2) {
12067 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12068 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12069 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12070 emit_int16((unsigned char)0xF5, (0xC0 | encode));
12071 }
12072
12073 void Assembler::pextl(Register dst, Register src1, Address src2) {
12074 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12075 InstructionMark im(this);
12076 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12077 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12078 emit_int8((unsigned char)0xF5);
12079 emit_operand(dst, src2, 0);
12080 }
12081
12082 void Assembler::pdepl(Register dst, Register src1, Address src2) {
12083 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12084 InstructionMark im(this);
12085 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12086 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12087 emit_int8((unsigned char)0xF5);
12088 emit_operand(dst, src2, 0);
12089 }
12090
12091 void Assembler::pextq(Register dst, Register src1, Address src2) {
12092 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12093 InstructionMark im(this);
12094 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12095 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12096 emit_int8((unsigned char)0xF5);
12097 emit_operand(dst, src2, 0);
12098 }
12099
12100 void Assembler::pdepq(Register dst, Register src1, Address src2) {
12101 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
12102 InstructionMark im(this);
12103 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12104 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12105 emit_int8((unsigned char)0xF5);
12106 emit_operand(dst, src2, 0);
12107 }
12108
12109 void Assembler::sarxl(Register dst, Register src1, Register src2) {
12110 assert(VM_Version::supports_bmi2(), "");
12111 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12112 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12113 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12114 }
12115
12116 void Assembler::sarxl(Register dst, Address src1, Register src2) {
12117 assert(VM_Version::supports_bmi2(), "");
12118 InstructionMark im(this);
12119 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12120 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12121 emit_int8((unsigned char)0xF7);
12122 emit_operand(dst, src1, 0);
12123 }
12124
12125 void Assembler::sarxq(Register dst, Register src1, Register src2) {
12126 assert(VM_Version::supports_bmi2(), "");
12127 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12128 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12129 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12130 }
12131
12132 void Assembler::sarxq(Register dst, Address src1, Register src2) {
12133 assert(VM_Version::supports_bmi2(), "");
12134 InstructionMark im(this);
12135 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12136 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12137 emit_int8((unsigned char)0xF7);
12138 emit_operand(dst, src1, 0);
12139 }
12140
12141 void Assembler::shlxl(Register dst, Register src1, Register src2) {
12142 assert(VM_Version::supports_bmi2(), "");
12143 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12144 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12145 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12146 }
12147
12148 void Assembler::shlxl(Register dst, Address src1, Register src2) {
12149 assert(VM_Version::supports_bmi2(), "");
12150 InstructionMark im(this);
12151 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12152 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12153 emit_int8((unsigned char)0xF7);
12154 emit_operand(dst, src1, 0);
12155 }
12156
12157 void Assembler::shlxq(Register dst, Register src1, Register src2) {
12158 assert(VM_Version::supports_bmi2(), "");
12159 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12160 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12161 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12162 }
12163
12164 void Assembler::shlxq(Register dst, Address src1, Register src2) {
12165 assert(VM_Version::supports_bmi2(), "");
12166 InstructionMark im(this);
12167 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12168 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12169 emit_int8((unsigned char)0xF7);
12170 emit_operand(dst, src1, 0);
12171 }
12172
12173 void Assembler::shrxl(Register dst, Register src1, Register src2) {
12174 assert(VM_Version::supports_bmi2(), "");
12175 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12176 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12177 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12178 }
12179
12180 void Assembler::shrxl(Register dst, Address src1, Register src2) {
12181 assert(VM_Version::supports_bmi2(), "");
12182 InstructionMark im(this);
12183 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12184 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12185 emit_int8((unsigned char)0xF7);
12186 emit_operand(dst, src1, 0);
12187 }
12188
12189 void Assembler::shrxq(Register dst, Register src1, Register src2) {
12190 assert(VM_Version::supports_bmi2(), "");
12191 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12192 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12193 emit_int16((unsigned char)0xF7, (0xC0 | encode));
12194 }
12195
12196 void Assembler::shrxq(Register dst, Address src1, Register src2) {
12197 assert(VM_Version::supports_bmi2(), "");
12198 InstructionMark im(this);
12199 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
12200 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
12201 emit_int8((unsigned char)0xF7);
12202 emit_operand(dst, src1, 0);
12203 }
12204
12205 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
12206 assert(VM_Version::supports_avx512vldq(), "");
12207 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12208 attributes.set_is_evex_instruction();
12209 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12210 emit_int16(0x39, (0xC0 | encode));
12211 }
12212
12213 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
12214 assert(VM_Version::supports_avx512vldq(), "");
12215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12216 attributes.set_is_evex_instruction();
12217 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12218 emit_int16(0x39, (0xC0 | encode));
12219 }
12220
12221 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
12222 assert(VM_Version::supports_avx512vlbw(), "");
12223 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12224 attributes.set_is_evex_instruction();
12225 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12226 emit_int16(0x29, (0xC0 | encode));
12227 }
12228
12229 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
12230 assert(VM_Version::supports_avx512vlbw(), "");
12231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12232 attributes.set_is_evex_instruction();
12233 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12234 emit_int16(0x29, (0xC0 | encode));
12235 }
12236
12237 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
12238 assert(VM_Version::supports_avx512vldq(), "");
12239 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12240 attributes.set_is_evex_instruction();
12241 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12242 emit_int16(0x38, (0xC0 | encode));
12243 }
12244
12245 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
12246 assert(VM_Version::supports_avx512vldq(), "");
12247 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12248 attributes.set_is_evex_instruction();
12249 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12250 emit_int16(0x38, (0xC0 | encode));
12251 }
12252
12253 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
12254 assert(VM_Version::supports_avx512vlbw(), "");
12255 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12256 attributes.set_is_evex_instruction();
12257 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12258 emit_int16(0x28, (0xC0 | encode));
12259 }
12260
12261 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
12262 assert(VM_Version::supports_avx512vlbw(), "");
12263 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
12264 attributes.set_is_evex_instruction();
12265 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
12266 emit_int16(0x28, (0xC0 | encode));
12267 }
12268
12269 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12270 assert(VM_Version::supports_avx512_vbmi2(), "");
12271 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12272 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12273 attributes.set_embedded_opmask_register_specifier(mask);
12274 attributes.set_is_evex_instruction();
12275 if (merge) {
12276 attributes.reset_is_clear_context();
12277 }
12278 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12279 emit_int16((unsigned char)0x63, (0xC0 | encode));
12280 }
12281
12282 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12283 assert(VM_Version::supports_avx512_vbmi2(), "");
12284 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12285 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12286 attributes.set_embedded_opmask_register_specifier(mask);
12287 attributes.set_is_evex_instruction();
12288 if (merge) {
12289 attributes.reset_is_clear_context();
12290 }
12291 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12292 emit_int16((unsigned char)0x63, (0xC0 | encode));
12293 }
12294
12295 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12296 assert(VM_Version::supports_evex(), "");
12297 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12299 attributes.set_embedded_opmask_register_specifier(mask);
12300 attributes.set_is_evex_instruction();
12301 if (merge) {
12302 attributes.reset_is_clear_context();
12303 }
12304 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12305 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12306 }
12307
12308 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12309 assert(VM_Version::supports_evex(), "");
12310 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12311 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12312 attributes.set_embedded_opmask_register_specifier(mask);
12313 attributes.set_is_evex_instruction();
12314 if (merge) {
12315 attributes.reset_is_clear_context();
12316 }
12317 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12318 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12319 }
12320
12321 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12322 assert(VM_Version::supports_evex(), "");
12323 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12324 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12325 attributes.set_embedded_opmask_register_specifier(mask);
12326 attributes.set_is_evex_instruction();
12327 if (merge) {
12328 attributes.reset_is_clear_context();
12329 }
12330 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12331 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12332 }
12333
12334 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12335 assert(VM_Version::supports_evex(), "");
12336 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12337 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12338 attributes.set_embedded_opmask_register_specifier(mask);
12339 attributes.set_is_evex_instruction();
12340 if (merge) {
12341 attributes.reset_is_clear_context();
12342 }
12343 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12344 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12345 }
12346
12347 #ifndef _LP64
12348
12349 void Assembler::incl(Register dst) {
12350 // Don't use it directly. Use MacroAssembler::incrementl() instead.
12351 emit_int8(0x40 | dst->encoding());
12352 }
12353
12354 void Assembler::lea(Register dst, Address src) {
12355 leal(dst, src);
12356 }
12357
12358 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12359 InstructionMark im(this);
12360 emit_int8((unsigned char)0xC7);
12361 emit_operand(rax, dst, 4);
12362 emit_data((int)imm32, rspec, 0);
12363 }
12364
12365 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12366 InstructionMark im(this);
12367 int encode = prefix_and_encode(dst->encoding());
12368 emit_int8((0xB8 | encode));
12369 emit_data((int)imm32, rspec, 0);
12370 }
12371
12372 void Assembler::popa() { // 32bit
12373 emit_int8(0x61);
12374 }
12375
12376 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12377 InstructionMark im(this);
12378 emit_int8(0x68);
12379 emit_data(imm32, rspec, 0);
12380 }
12381
12382 void Assembler::pusha() { // 32bit
12383 emit_int8(0x60);
12384 }
12385
12386 void Assembler::set_byte_if_not_zero(Register dst) {
12387 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12388 }
12389
12390 #else // LP64
12391
12392 // 64bit only pieces of the assembler
12393
12394 void Assembler::set_byte_if_not_zero(Register dst) {
12395 int enc = prefix_and_encode(dst->encoding(), true);
12396 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12397 }
12398
12399 // This should only be used by 64bit instructions that can use rip-relative
12400 // it cannot be used by instructions that want an immediate value.
12401
12402 // Determine whether an address is always reachable in rip-relative addressing mode
12403 // when accessed from the code cache.
12404 static bool is_always_reachable(address target, relocInfo::relocType reloc_type) {
12405 switch (reloc_type) {
12406 // This should be rip-relative and easily reachable.
12407 case relocInfo::internal_word_type: {
12408 return true;
12409 }
12410 // This should be rip-relative within the code cache and easily
12411 // reachable until we get huge code caches. (At which point
12412 // IC code is going to have issues).
12413 case relocInfo::virtual_call_type:
12414 case relocInfo::opt_virtual_call_type:
12415 case relocInfo::static_call_type:
12416 case relocInfo::static_stub_type: {
12417 return true;
12418 }
12419 case relocInfo::runtime_call_type:
12420 case relocInfo::external_word_type:
12421 case relocInfo::poll_return_type: // these are really external_word but need special
12422 case relocInfo::poll_type: { // relocs to identify them
12423 return CodeCache::contains(target);
12424 }
12425 default: {
12426 return false;
12427 }
12428 }
12429 }
12430
12431 // Determine whether an address is reachable in rip-relative addressing mode from the code cache.
12432 static bool is_reachable(address target, relocInfo::relocType reloc_type) {
12433 if (is_always_reachable(target, reloc_type)) {
12434 return true;
12435 }
12436 switch (reloc_type) {
12437 // None will force a 64bit literal to the code stream. Likely a placeholder
12438 // for something that will be patched later and we need to certain it will
12439 // always be reachable.
12440 case relocInfo::none: {
12441 return false;
12442 }
12443 case relocInfo::runtime_call_type:
12444 case relocInfo::external_word_type:
12445 case relocInfo::poll_return_type: // these are really external_word but need special
12446 case relocInfo::poll_type: { // relocs to identify them
12447 assert(!CodeCache::contains(target), "always reachable");
12448 if (ForceUnreachable) {
12449 return false; // stress the correction code
12450 }
12451 // For external_word_type/runtime_call_type if it is reachable from where we
12452 // are now (possibly a temp buffer) and where we might end up
12453 // anywhere in the code cache then we are always reachable.
12454 // This would have to change if we ever save/restore shared code to be more pessimistic.
12455 // Code buffer has to be allocated in the code cache, so check against
12456 // code cache boundaries cover that case.
12457 //
12458 // In rip-relative addressing mode, an effective address is formed by adding displacement
12459 // to the 64-bit RIP of the next instruction which is not known yet. Considering target address
12460 // is guaranteed to be outside of the code cache, checking against code cache boundaries is enough
12461 // to account for that.
12462 return Assembler::is_simm32(target - CodeCache::low_bound()) &&
12463 Assembler::is_simm32(target - CodeCache::high_bound());
12464 }
12465 default: {
12466 return false;
12467 }
12468 }
12469 }
12470
12471 bool Assembler::reachable(AddressLiteral adr) {
12472 assert(CodeCache::contains(pc()), "required");
12473 if (adr.is_lval()) {
12474 return false;
12475 }
12476 return is_reachable(adr.target(), adr.reloc());
12477 }
12478
12479 bool Assembler::always_reachable(AddressLiteral adr) {
12480 assert(CodeCache::contains(pc()), "required");
12481 if (adr.is_lval()) {
12482 return false;
12483 }
12484 return is_always_reachable(adr.target(), adr.reloc());
12485 }
12486
12487 void Assembler::emit_data64(jlong data,
12488 relocInfo::relocType rtype,
12489 int format) {
12490 if (rtype == relocInfo::none) {
12491 emit_int64(data);
12492 } else {
12493 emit_data64(data, Relocation::spec_simple(rtype), format);
12494 }
12495 }
12496
12497 void Assembler::emit_data64(jlong data,
12498 RelocationHolder const& rspec,
12499 int format) {
12500 assert(imm_operand == 0, "default format must be immediate in this file");
12501 assert(imm_operand == format, "must be immediate");
12502 assert(inst_mark() != NULL, "must be inside InstructionMark");
12503 // Do not use AbstractAssembler::relocate, which is not intended for
12504 // embedded words. Instead, relocate to the enclosing instruction.
12505 code_section()->relocate(inst_mark(), rspec, format);
12506 #ifdef ASSERT
12507 check_relocation(rspec, format);
12508 #endif
12509 emit_int64(data);
12510 }
12511
12512 void Assembler::prefix(Register reg) {
12513 if (reg->encoding() >= 8) {
12514 prefix(REX_B);
12515 }
12516 }
12517
12518 void Assembler::prefix(Register dst, Register src, Prefix p) {
12519 if (src->encoding() >= 8) {
12520 p = (Prefix)(p | REX_B);
12521 }
12522 if (dst->encoding() >= 8) {
12523 p = (Prefix)(p | REX_R);
12524 }
12525 if (p != Prefix_EMPTY) {
12526 // do not generate an empty prefix
12527 prefix(p);
12528 }
12529 }
12530
12531 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12532 if (adr.base_needs_rex()) {
12533 if (adr.index_needs_rex()) {
12534 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12535 } else {
12536 prefix(REX_B);
12537 }
12538 } else {
12539 if (adr.index_needs_rex()) {
12540 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12541 }
12542 }
12543 if (dst->encoding() >= 8) {
12544 p = (Prefix)(p | REX_R);
12545 }
12546 if (p != Prefix_EMPTY) {
12547 // do not generate an empty prefix
12548 prefix(p);
12549 }
12550 }
12551
12552 void Assembler::prefix(Address adr) {
12553 if (adr.base_needs_rex()) {
12554 if (adr.index_needs_rex()) {
12555 prefix(REX_XB);
12556 } else {
12557 prefix(REX_B);
12558 }
12559 } else {
12560 if (adr.index_needs_rex()) {
12561 prefix(REX_X);
12562 }
12563 }
12564 }
12565
12566 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12567 if (reg->encoding() < 8) {
12568 if (adr.base_needs_rex()) {
12569 if (adr.index_needs_rex()) {
12570 prefix(REX_XB);
12571 } else {
12572 prefix(REX_B);
12573 }
12574 } else {
12575 if (adr.index_needs_rex()) {
12576 prefix(REX_X);
12577 } else if (byteinst && reg->encoding() >= 4) {
12578 prefix(REX);
12579 }
12580 }
12581 } else {
12582 if (adr.base_needs_rex()) {
12583 if (adr.index_needs_rex()) {
12584 prefix(REX_RXB);
12585 } else {
12586 prefix(REX_RB);
12587 }
12588 } else {
12589 if (adr.index_needs_rex()) {
12590 prefix(REX_RX);
12591 } else {
12592 prefix(REX_R);
12593 }
12594 }
12595 }
12596 }
12597
12598 void Assembler::prefix(Address adr, XMMRegister reg) {
12599 if (reg->encoding() < 8) {
12600 if (adr.base_needs_rex()) {
12601 if (adr.index_needs_rex()) {
12602 prefix(REX_XB);
12603 } else {
12604 prefix(REX_B);
12605 }
12606 } else {
12607 if (adr.index_needs_rex()) {
12608 prefix(REX_X);
12609 }
12610 }
12611 } else {
12612 if (adr.base_needs_rex()) {
12613 if (adr.index_needs_rex()) {
12614 prefix(REX_RXB);
12615 } else {
12616 prefix(REX_RB);
12617 }
12618 } else {
12619 if (adr.index_needs_rex()) {
12620 prefix(REX_RX);
12621 } else {
12622 prefix(REX_R);
12623 }
12624 }
12625 }
12626 }
12627
12628 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12629 if (reg_enc >= 8) {
12630 prefix(REX_B);
12631 reg_enc -= 8;
12632 } else if (byteinst && reg_enc >= 4) {
12633 prefix(REX);
12634 }
12635 return reg_enc;
12636 }
12637
12638 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12639 if (dst_enc < 8) {
12640 if (src_enc >= 8) {
12641 prefix(REX_B);
12642 src_enc -= 8;
12643 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12644 prefix(REX);
12645 }
12646 } else {
12647 if (src_enc < 8) {
12648 prefix(REX_R);
12649 } else {
12650 prefix(REX_RB);
12651 src_enc -= 8;
12652 }
12653 dst_enc -= 8;
12654 }
12655 return dst_enc << 3 | src_enc;
12656 }
12657
12658 int8_t Assembler::get_prefixq(Address adr) {
12659 int8_t prfx = get_prefixq(adr, rax);
12660 assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12661 return prfx;
12662 }
12663
12664 int8_t Assembler::get_prefixq(Address adr, Register src) {
12665 int8_t prfx = (int8_t)(REX_W +
12666 ((int)adr.base_needs_rex()) +
12667 ((int)adr.index_needs_rex() << 1) +
12668 ((int)(src->encoding() >= 8) << 2));
12669 #ifdef ASSERT
12670 if (src->encoding() < 8) {
12671 if (adr.base_needs_rex()) {
12672 if (adr.index_needs_rex()) {
12673 assert(prfx == REX_WXB, "must be");
12674 } else {
12675 assert(prfx == REX_WB, "must be");
12676 }
12677 } else {
12678 if (adr.index_needs_rex()) {
12679 assert(prfx == REX_WX, "must be");
12680 } else {
12681 assert(prfx == REX_W, "must be");
12682 }
12683 }
12684 } else {
12685 if (adr.base_needs_rex()) {
12686 if (adr.index_needs_rex()) {
12687 assert(prfx == REX_WRXB, "must be");
12688 } else {
12689 assert(prfx == REX_WRB, "must be");
12690 }
12691 } else {
12692 if (adr.index_needs_rex()) {
12693 assert(prfx == REX_WRX, "must be");
12694 } else {
12695 assert(prfx == REX_WR, "must be");
12696 }
12697 }
12698 }
12699 #endif
12700 return prfx;
12701 }
12702
12703 void Assembler::prefixq(Address adr) {
12704 emit_int8(get_prefixq(adr));
12705 }
12706
12707 void Assembler::prefixq(Address adr, Register src) {
12708 emit_int8(get_prefixq(adr, src));
12709 }
12710
12711 void Assembler::prefixq(Address adr, XMMRegister src) {
12712 if (src->encoding() < 8) {
12713 if (adr.base_needs_rex()) {
12714 if (adr.index_needs_rex()) {
12715 prefix(REX_WXB);
12716 } else {
12717 prefix(REX_WB);
12718 }
12719 } else {
12720 if (adr.index_needs_rex()) {
12721 prefix(REX_WX);
12722 } else {
12723 prefix(REX_W);
12724 }
12725 }
12726 } else {
12727 if (adr.base_needs_rex()) {
12728 if (adr.index_needs_rex()) {
12729 prefix(REX_WRXB);
12730 } else {
12731 prefix(REX_WRB);
12732 }
12733 } else {
12734 if (adr.index_needs_rex()) {
12735 prefix(REX_WRX);
12736 } else {
12737 prefix(REX_WR);
12738 }
12739 }
12740 }
12741 }
12742
12743 int Assembler::prefixq_and_encode(int reg_enc) {
12744 if (reg_enc < 8) {
12745 prefix(REX_W);
12746 } else {
12747 prefix(REX_WB);
12748 reg_enc -= 8;
12749 }
12750 return reg_enc;
12751 }
12752
12753 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12754 if (dst_enc < 8) {
12755 if (src_enc < 8) {
12756 prefix(REX_W);
12757 } else {
12758 prefix(REX_WB);
12759 src_enc -= 8;
12760 }
12761 } else {
12762 if (src_enc < 8) {
12763 prefix(REX_WR);
12764 } else {
12765 prefix(REX_WRB);
12766 src_enc -= 8;
12767 }
12768 dst_enc -= 8;
12769 }
12770 return dst_enc << 3 | src_enc;
12771 }
12772
12773 void Assembler::adcq(Register dst, int32_t imm32) {
12774 (void) prefixq_and_encode(dst->encoding());
12775 emit_arith(0x81, 0xD0, dst, imm32);
12776 }
12777
12778 void Assembler::adcq(Register dst, Address src) {
12779 InstructionMark im(this);
12780 emit_int16(get_prefixq(src, dst), 0x13);
12781 emit_operand(dst, src, 0);
12782 }
12783
12784 void Assembler::adcq(Register dst, Register src) {
12785 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12786 emit_arith(0x13, 0xC0, dst, src);
12787 }
12788
12789 void Assembler::addq(Address dst, int32_t imm32) {
12790 InstructionMark im(this);
12791 prefixq(dst);
12792 emit_arith_operand(0x81, rax, dst, imm32);
12793 }
12794
12795 void Assembler::addq(Address dst, Register src) {
12796 InstructionMark im(this);
12797 emit_int16(get_prefixq(dst, src), 0x01);
12798 emit_operand(src, dst, 0);
12799 }
12800
12801 void Assembler::addq(Register dst, int32_t imm32) {
12802 (void) prefixq_and_encode(dst->encoding());
12803 emit_arith(0x81, 0xC0, dst, imm32);
12804 }
12805
12806 void Assembler::addq(Register dst, Address src) {
12807 InstructionMark im(this);
12808 emit_int16(get_prefixq(src, dst), 0x03);
12809 emit_operand(dst, src, 0);
12810 }
12811
12812 void Assembler::addq(Register dst, Register src) {
12813 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12814 emit_arith(0x03, 0xC0, dst, src);
12815 }
12816
12817 void Assembler::adcxq(Register dst, Register src) {
12818 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12819 emit_int8(0x66);
12820 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12821 emit_int32(0x0F,
12822 0x38,
12823 (unsigned char)0xF6,
12824 (0xC0 | encode));
12825 }
12826
12827 void Assembler::adoxq(Register dst, Register src) {
12828 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12829 emit_int8((unsigned char)0xF3);
12830 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12831 emit_int32(0x0F,
12832 0x38,
12833 (unsigned char)0xF6,
12834 (0xC0 | encode));
12835 }
12836
12837 void Assembler::andq(Address dst, int32_t imm32) {
12838 InstructionMark im(this);
12839 prefixq(dst);
12840 emit_arith_operand(0x81, as_Register(4), dst, imm32);
12841 }
12842
12843 void Assembler::andq(Register dst, int32_t imm32) {
12844 (void) prefixq_and_encode(dst->encoding());
12845 emit_arith(0x81, 0xE0, dst, imm32);
12846 }
12847
12848 void Assembler::andq(Register dst, Address src) {
12849 InstructionMark im(this);
12850 emit_int16(get_prefixq(src, dst), 0x23);
12851 emit_operand(dst, src, 0);
12852 }
12853
12854 void Assembler::andq(Register dst, Register src) {
12855 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12856 emit_arith(0x23, 0xC0, dst, src);
12857 }
12858
12859 void Assembler::andq(Address dst, Register src) {
12860 InstructionMark im(this);
12861 emit_int16(get_prefixq(dst, src), 0x21);
12862 emit_operand(src, dst, 0);
12863 }
12864
12865 void Assembler::andnq(Register dst, Register src1, Register src2) {
12866 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12867 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12868 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12869 emit_int16((unsigned char)0xF2, (0xC0 | encode));
12870 }
12871
12872 void Assembler::andnq(Register dst, Register src1, Address src2) {
12873 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12874 InstructionMark im(this);
12875 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12876 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12877 emit_int8((unsigned char)0xF2);
12878 emit_operand(dst, src2, 0);
12879 }
12880
12881 void Assembler::bsfq(Register dst, Register src) {
12882 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12883 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12884 }
12885
12886 void Assembler::bsrq(Register dst, Register src) {
12887 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12888 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12889 }
12890
12891 void Assembler::bswapq(Register reg) {
12892 int encode = prefixq_and_encode(reg->encoding());
12893 emit_int16(0x0F, (0xC8 | encode));
12894 }
12895
12896 void Assembler::blsiq(Register dst, Register src) {
12897 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12898 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12899 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12900 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12901 }
12902
12903 void Assembler::blsiq(Register dst, Address src) {
12904 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12905 InstructionMark im(this);
12906 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12907 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12908 emit_int8((unsigned char)0xF3);
12909 emit_operand(rbx, src, 0);
12910 }
12911
12912 void Assembler::blsmskq(Register dst, Register src) {
12913 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12914 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12915 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12916 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12917 }
12918
12919 void Assembler::blsmskq(Register dst, Address src) {
12920 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12921 InstructionMark im(this);
12922 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12923 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12924 emit_int8((unsigned char)0xF3);
12925 emit_operand(rdx, src, 0);
12926 }
12927
12928 void Assembler::blsrq(Register dst, Register src) {
12929 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12930 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12931 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12932 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12933 }
12934
12935 void Assembler::blsrq(Register dst, Address src) {
12936 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12937 InstructionMark im(this);
12938 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12939 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12940 emit_int8((unsigned char)0xF3);
12941 emit_operand(rcx, src, 0);
12942 }
12943
12944 void Assembler::cdqq() {
12945 emit_int16(REX_W, (unsigned char)0x99);
12946 }
12947
12948 void Assembler::clflush(Address adr) {
12949 assert(VM_Version::supports_clflush(), "should do");
12950 prefix(adr);
12951 emit_int16(0x0F, (unsigned char)0xAE);
12952 emit_operand(rdi, adr, 0);
12953 }
12954
12955 void Assembler::clflushopt(Address adr) {
12956 assert(VM_Version::supports_clflushopt(), "should do!");
12957 // adr should be base reg only with no index or offset
12958 assert(adr.index() == noreg, "index should be noreg");
12959 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12960 assert(adr.disp() == 0, "displacement should be 0");
12961 // instruction prefix is 0x66
12962 emit_int8(0x66);
12963 prefix(adr);
12964 // opcode family is 0x0F 0xAE
12965 emit_int16(0x0F, (unsigned char)0xAE);
12966 // extended opcode byte is 7 == rdi
12967 emit_operand(rdi, adr, 0);
12968 }
12969
12970 void Assembler::clwb(Address adr) {
12971 assert(VM_Version::supports_clwb(), "should do!");
12972 // adr should be base reg only with no index or offset
12973 assert(adr.index() == noreg, "index should be noreg");
12974 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12975 assert(adr.disp() == 0, "displacement should be 0");
12976 // instruction prefix is 0x66
12977 emit_int8(0x66);
12978 prefix(adr);
12979 // opcode family is 0x0f 0xAE
12980 emit_int16(0x0F, (unsigned char)0xAE);
12981 // extended opcode byte is 6 == rsi
12982 emit_operand(rsi, adr, 0);
12983 }
12984
12985 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12986 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12987 emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12988 }
12989
12990 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12991 InstructionMark im(this);
12992 emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12993 emit_operand(dst, src, 0);
12994 }
12995
12996 void Assembler::cmpq(Address dst, int32_t imm32) {
12997 InstructionMark im(this);
12998 prefixq(dst);
12999 emit_arith_operand(0x81, as_Register(7), dst, imm32);
13000 }
13001
13002 void Assembler::cmpq(Register dst, int32_t imm32) {
13003 (void) prefixq_and_encode(dst->encoding());
13004 emit_arith(0x81, 0xF8, dst, imm32);
13005 }
13006
13007 void Assembler::cmpq(Address dst, Register src) {
13008 InstructionMark im(this);
13009 emit_int16(get_prefixq(dst, src), 0x39);
13010 emit_operand(src, dst, 0);
13011 }
13012
13013 void Assembler::cmpq(Register dst, Register src) {
13014 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13015 emit_arith(0x3B, 0xC0, dst, src);
13016 }
13017
13018 void Assembler::cmpq(Register dst, Address src) {
13019 InstructionMark im(this);
13020 emit_int16(get_prefixq(src, dst), 0x3B);
13021 emit_operand(dst, src, 0);
13022 }
13023
13024 void Assembler::cmpxchgq(Register reg, Address adr) {
13025 InstructionMark im(this);
13026 emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
13027 emit_operand(reg, adr, 0);
13028 }
13029
13030 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
13031 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13032 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13033 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13034 emit_int16(0x2A, (0xC0 | encode));
13035 }
13036
13037 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
13038 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13039 InstructionMark im(this);
13040 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13041 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13042 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13043 emit_int8(0x2A);
13044 emit_operand(dst, src, 0);
13045 }
13046
13047 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
13048 NOT_LP64(assert(VM_Version::supports_sse(), ""));
13049 InstructionMark im(this);
13050 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13051 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
13052 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13053 emit_int8(0x2A);
13054 emit_operand(dst, src, 0);
13055 }
13056
13057 void Assembler::cvttsd2siq(Register dst, Address src) {
13058 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13059 // F2 REX.W 0F 2C /r
13060 // CVTTSD2SI r64, xmm1/m64
13061 InstructionMark im(this);
13062 emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
13063 emit_operand(dst, src, 0);
13064 }
13065
13066 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
13067 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13068 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13069 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13070 emit_int16(0x2C, (0xC0 | encode));
13071 }
13072
13073 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
13074 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13075 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13076 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
13077 emit_int16(0x2D, (0xC0 | encode));
13078 }
13079
13080 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
13081 NOT_LP64(assert(VM_Version::supports_sse(), ""));
13082 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13083 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
13084 emit_int16(0x2C, (0xC0 | encode));
13085 }
13086
13087 void Assembler::decl(Register dst) {
13088 // Don't use it directly. Use MacroAssembler::decrementl() instead.
13089 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
13090 int encode = prefix_and_encode(dst->encoding());
13091 emit_int16((unsigned char)0xFF, (0xC8 | encode));
13092 }
13093
13094 void Assembler::decq(Register dst) {
13095 // Don't use it directly. Use MacroAssembler::decrementq() instead.
13096 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13097 int encode = prefixq_and_encode(dst->encoding());
13098 emit_int16((unsigned char)0xFF, 0xC8 | encode);
13099 }
13100
13101 void Assembler::decq(Address dst) {
13102 // Don't use it directly. Use MacroAssembler::decrementq() instead.
13103 InstructionMark im(this);
13104 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13105 emit_operand(rcx, dst, 0);
13106 }
13107
13108 void Assembler::fxrstor(Address src) {
13109 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13110 emit_operand(as_Register(1), src, 0);
13111 }
13112
13113 void Assembler::xrstor(Address src) {
13114 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
13115 emit_operand(as_Register(5), src, 0);
13116 }
13117
13118 void Assembler::fxsave(Address dst) {
13119 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13120 emit_operand(as_Register(0), dst, 0);
13121 }
13122
13123 void Assembler::xsave(Address dst) {
13124 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
13125 emit_operand(as_Register(4), dst, 0);
13126 }
13127
13128 void Assembler::idivq(Register src) {
13129 int encode = prefixq_and_encode(src->encoding());
13130 emit_int16((unsigned char)0xF7, (0xF8 | encode));
13131 }
13132
13133 void Assembler::divq(Register src) {
13134 int encode = prefixq_and_encode(src->encoding());
13135 emit_int16((unsigned char)0xF7, (0xF0 | encode));
13136 }
13137
13138 void Assembler::imulq(Register dst, Register src) {
13139 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13140 emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
13141 }
13142
13143 void Assembler::imulq(Register src) {
13144 int encode = prefixq_and_encode(src->encoding());
13145 emit_int16((unsigned char)0xF7, (0xE8 | encode));
13146 }
13147
13148 void Assembler::imulq(Register dst, Address src, int32_t value) {
13149 InstructionMark im(this);
13150 prefixq(src, dst);
13151 if (is8bit(value)) {
13152 emit_int8((unsigned char)0x6B);
13153 emit_operand(dst, src, 1);
13154 emit_int8(value);
13155 } else {
13156 emit_int8((unsigned char)0x69);
13157 emit_operand(dst, src, 4);
13158 emit_int32(value);
13159 }
13160 }
13161
13162 void Assembler::imulq(Register dst, Register src, int value) {
13163 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13164 if (is8bit(value)) {
13165 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
13166 } else {
13167 emit_int16(0x69, (0xC0 | encode));
13168 emit_int32(value);
13169 }
13170 }
13171
13172 void Assembler::imulq(Register dst, Address src) {
13173 InstructionMark im(this);
13174 emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
13175 emit_operand(dst, src, 0);
13176 }
13177
13178 void Assembler::incl(Register dst) {
13179 // Don't use it directly. Use MacroAssembler::incrementl() instead.
13180 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13181 int encode = prefix_and_encode(dst->encoding());
13182 emit_int16((unsigned char)0xFF, (0xC0 | encode));
13183 }
13184
13185 void Assembler::incq(Register dst) {
13186 // Don't use it directly. Use MacroAssembler::incrementq() instead.
13187 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
13188 int encode = prefixq_and_encode(dst->encoding());
13189 emit_int16((unsigned char)0xFF, (0xC0 | encode));
13190 }
13191
13192 void Assembler::incq(Address dst) {
13193 // Don't use it directly. Use MacroAssembler::incrementq() instead.
13194 InstructionMark im(this);
13195 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
13196 emit_operand(rax, dst, 0);
13197 }
13198
13199 void Assembler::lea(Register dst, Address src) {
13200 leaq(dst, src);
13201 }
13202
13203 void Assembler::leaq(Register dst, Address src) {
13204 InstructionMark im(this);
13205 emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
13206 emit_operand(dst, src, 0);
13207 }
13208
13209 void Assembler::mov64(Register dst, int64_t imm64) {
13210 InstructionMark im(this);
13211 int encode = prefixq_and_encode(dst->encoding());
13212 emit_int8(0xB8 | encode);
13213 emit_int64(imm64);
13214 }
13215
13216 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
13217 InstructionMark im(this);
13218 int encode = prefixq_and_encode(dst->encoding());
13219 emit_int8(0xB8 | encode);
13220 emit_data64(imm64, rtype, format);
13221 }
13222
13223 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
13224 InstructionMark im(this);
13225 int encode = prefixq_and_encode(dst->encoding());
13226 emit_int8(0xB8 | encode);
13227 emit_data64(imm64, rspec);
13228 }
13229
13230 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
13231 InstructionMark im(this);
13232 int encode = prefix_and_encode(dst->encoding());
13233 emit_int8(0xB8 | encode);
13234 emit_data((int)imm32, rspec, narrow_oop_operand);
13235 }
13236
13237 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
13238 InstructionMark im(this);
13239 prefix(dst);
13240 emit_int8((unsigned char)0xC7);
13241 emit_operand(rax, dst, 4);
13242 emit_data((int)imm32, rspec, narrow_oop_operand);
13243 }
13244
13245 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
13246 InstructionMark im(this);
13247 int encode = prefix_and_encode(src1->encoding());
13248 emit_int16((unsigned char)0x81, (0xF8 | encode));
13249 emit_data((int)imm32, rspec, narrow_oop_operand);
13250 }
13251
13252 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
13253 InstructionMark im(this);
13254 prefix(src1);
13255 emit_int8((unsigned char)0x81);
13256 emit_operand(rax, src1, 4);
13257 emit_data((int)imm32, rspec, narrow_oop_operand);
13258 }
13259
13260 void Assembler::lzcntq(Register dst, Register src) {
13261 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13262 emit_int8((unsigned char)0xF3);
13263 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13264 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
13265 }
13266
13267 void Assembler::lzcntq(Register dst, Address src) {
13268 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
13269 InstructionMark im(this);
13270 emit_int8((unsigned char)0xF3);
13271 prefixq(src, dst);
13272 emit_int16(0x0F, (unsigned char)0xBD);
13273 emit_operand(dst, src, 0);
13274 }
13275
13276 void Assembler::movdq(XMMRegister dst, Register src) {
13277 // table D-1 says MMX/SSE2
13278 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13279 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13280 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13281 emit_int16(0x6E, (0xC0 | encode));
13282 }
13283
13284 void Assembler::movdq(Register dst, XMMRegister src) {
13285 // table D-1 says MMX/SSE2
13286 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
13287 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
13288 // swap src/dst to get correct prefix
13289 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
13290 emit_int16(0x7E,
13291 (0xC0 | encode));
13292 }
13293
13294 void Assembler::movq(Register dst, Register src) {
13295 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13296 emit_int16((unsigned char)0x8B,
13297 (0xC0 | encode));
13298 }
13299
13300 void Assembler::movq(Register dst, Address src) {
13301 InstructionMark im(this);
13302 emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
13303 emit_operand(dst, src, 0);
13304 }
13305
13306 void Assembler::movq(Address dst, Register src) {
13307 InstructionMark im(this);
13308 emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
13309 emit_operand(src, dst, 0);
13310 }
13311
13312 void Assembler::movq(Address dst, int32_t imm32) {
13313 InstructionMark im(this);
13314 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13315 emit_operand(as_Register(0), dst, 4);
13316 emit_int32(imm32);
13317 }
13318
13319 void Assembler::movq(Register dst, int32_t imm32) {
13320 int encode = prefixq_and_encode(dst->encoding());
13321 emit_int16((unsigned char)0xC7, (0xC0 | encode));
13322 emit_int32(imm32);
13323 }
13324
13325 void Assembler::movsbq(Register dst, Address src) {
13326 InstructionMark im(this);
13327 emit_int24(get_prefixq(src, dst),
13328 0x0F,
13329 (unsigned char)0xBE);
13330 emit_operand(dst, src, 0);
13331 }
13332
13333 void Assembler::movsbq(Register dst, Register src) {
13334 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13335 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13336 }
13337
13338 void Assembler::movslq(Register dst, int32_t imm32) {
13339 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
13340 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
13341 // as a result we shouldn't use until tested at runtime...
13342 ShouldNotReachHere();
13343 InstructionMark im(this);
13344 int encode = prefixq_and_encode(dst->encoding());
13345 emit_int8(0xC7 | encode);
13346 emit_int32(imm32);
13347 }
13348
13349 void Assembler::movslq(Address dst, int32_t imm32) {
13350 assert(is_simm32(imm32), "lost bits");
13351 InstructionMark im(this);
13352 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13353 emit_operand(rax, dst, 4);
13354 emit_int32(imm32);
13355 }
13356
13357 void Assembler::movslq(Register dst, Address src) {
13358 InstructionMark im(this);
13359 emit_int16(get_prefixq(src, dst), 0x63);
13360 emit_operand(dst, src, 0);
13361 }
13362
13363 void Assembler::movslq(Register dst, Register src) {
13364 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13365 emit_int16(0x63, (0xC0 | encode));
13366 }
13367
13368 void Assembler::movswq(Register dst, Address src) {
13369 InstructionMark im(this);
13370 emit_int24(get_prefixq(src, dst),
13371 0x0F,
13372 (unsigned char)0xBF);
13373 emit_operand(dst, src, 0);
13374 }
13375
13376 void Assembler::movswq(Register dst, Register src) {
13377 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13378 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13379 }
13380
13381 void Assembler::movzbq(Register dst, Address src) {
13382 InstructionMark im(this);
13383 emit_int24(get_prefixq(src, dst),
13384 0x0F,
13385 (unsigned char)0xB6);
13386 emit_operand(dst, src, 0);
13387 }
13388
13389 void Assembler::movzbq(Register dst, Register src) {
13390 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13391 emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13392 }
13393
13394 void Assembler::movzwq(Register dst, Address src) {
13395 InstructionMark im(this);
13396 emit_int24(get_prefixq(src, dst),
13397 0x0F,
13398 (unsigned char)0xB7);
13399 emit_operand(dst, src, 0);
13400 }
13401
13402 void Assembler::movzwq(Register dst, Register src) {
13403 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13404 emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13405 }
13406
13407 void Assembler::mulq(Address src) {
13408 InstructionMark im(this);
13409 emit_int16(get_prefixq(src), (unsigned char)0xF7);
13410 emit_operand(rsp, src, 0);
13411 }
13412
13413 void Assembler::mulq(Register src) {
13414 int encode = prefixq_and_encode(src->encoding());
13415 emit_int16((unsigned char)0xF7, (0xE0 | encode));
13416 }
13417
13418 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13419 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13420 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13421 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13422 emit_int16((unsigned char)0xF6, (0xC0 | encode));
13423 }
13424
13425 void Assembler::negq(Register dst) {
13426 int encode = prefixq_and_encode(dst->encoding());
13427 emit_int16((unsigned char)0xF7, (0xD8 | encode));
13428 }
13429
13430 void Assembler::negq(Address dst) {
13431 InstructionMark im(this);
13432 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13433 emit_operand(as_Register(3), dst, 0);
13434 }
13435
13436 void Assembler::notq(Register dst) {
13437 int encode = prefixq_and_encode(dst->encoding());
13438 emit_int16((unsigned char)0xF7, (0xD0 | encode));
13439 }
13440
13441 void Assembler::btsq(Address dst, int imm8) {
13442 assert(isByte(imm8), "not a byte");
13443 InstructionMark im(this);
13444 emit_int24(get_prefixq(dst),
13445 0x0F,
13446 (unsigned char)0xBA);
13447 emit_operand(rbp /* 5 */, dst, 1);
13448 emit_int8(imm8);
13449 }
13450
13451 void Assembler::btrq(Address dst, int imm8) {
13452 assert(isByte(imm8), "not a byte");
13453 InstructionMark im(this);
13454 emit_int24(get_prefixq(dst),
13455 0x0F,
13456 (unsigned char)0xBA);
13457 emit_operand(rsi /* 6 */, dst, 1);
13458 emit_int8(imm8);
13459 }
13460
13461 void Assembler::orq(Address dst, int32_t imm32) {
13462 InstructionMark im(this);
13463 prefixq(dst);
13464 emit_arith_operand(0x81, as_Register(1), dst, imm32);
13465 }
13466
13467 void Assembler::orq(Address dst, Register src) {
13468 InstructionMark im(this);
13469 emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13470 emit_operand(src, dst, 0);
13471 }
13472
13473 void Assembler::orq(Register dst, int32_t imm32) {
13474 (void) prefixq_and_encode(dst->encoding());
13475 emit_arith(0x81, 0xC8, dst, imm32);
13476 }
13477
13478 void Assembler::orq_imm32(Register dst, int32_t imm32) {
13479 (void) prefixq_and_encode(dst->encoding());
13480 emit_arith_imm32(0x81, 0xC8, dst, imm32);
13481 }
13482
13483 void Assembler::orq(Register dst, Address src) {
13484 InstructionMark im(this);
13485 emit_int16(get_prefixq(src, dst), 0x0B);
13486 emit_operand(dst, src, 0);
13487 }
13488
13489 void Assembler::orq(Register dst, Register src) {
13490 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13491 emit_arith(0x0B, 0xC0, dst, src);
13492 }
13493
13494 void Assembler::popcntq(Register dst, Address src) {
13495 assert(VM_Version::supports_popcnt(), "must support");
13496 InstructionMark im(this);
13497 emit_int32((unsigned char)0xF3,
13498 get_prefixq(src, dst),
13499 0x0F,
13500 (unsigned char)0xB8);
13501 emit_operand(dst, src, 0);
13502 }
13503
13504 void Assembler::popcntq(Register dst, Register src) {
13505 assert(VM_Version::supports_popcnt(), "must support");
13506 emit_int8((unsigned char)0xF3);
13507 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13508 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13509 }
13510
13511 void Assembler::popq(Address dst) {
13512 InstructionMark im(this);
13513 emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13514 emit_operand(rax, dst, 0);
13515 }
13516
13517 void Assembler::popq(Register dst) {
13518 emit_int8((unsigned char)0x58 | dst->encoding());
13519 }
13520
13521 // Precomputable: popa, pusha, vzeroupper
13522
13523 // The result of these routines are invariant from one invocation to another
13524 // invocation for the duration of a run. Caching the result on bootstrap
13525 // and copying it out on subsequent invocations can thus be beneficial
13526 static bool precomputed = false;
13527
13528 static u_char* popa_code = NULL;
13529 static int popa_len = 0;
13530
13531 static u_char* pusha_code = NULL;
13532 static int pusha_len = 0;
13533
13534 static u_char* vzup_code = NULL;
13535 static int vzup_len = 0;
13536
13537 void Assembler::precompute_instructions() {
13538 assert(!Universe::is_fully_initialized(), "must still be single threaded");
13539 guarantee(!precomputed, "only once");
13540 precomputed = true;
13541 ResourceMark rm;
13542
13543 // Make a temporary buffer big enough for the routines we're capturing
13544 int size = 256;
13545 char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13546 CodeBuffer buffer((address)tmp_code, size);
13547 MacroAssembler masm(&buffer);
13548
13549 address begin_popa = masm.code_section()->end();
13550 masm.popa_uncached();
13551 address end_popa = masm.code_section()->end();
13552 masm.pusha_uncached();
13553 address end_pusha = masm.code_section()->end();
13554 masm.vzeroupper_uncached();
13555 address end_vzup = masm.code_section()->end();
13556
13557 // Save the instructions to permanent buffers.
13558 popa_len = (int)(end_popa - begin_popa);
13559 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13560 memcpy(popa_code, begin_popa, popa_len);
13561
13562 pusha_len = (int)(end_pusha - end_popa);
13563 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13564 memcpy(pusha_code, end_popa, pusha_len);
13565
13566 vzup_len = (int)(end_vzup - end_pusha);
13567 if (vzup_len > 0) {
13568 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13569 memcpy(vzup_code, end_pusha, vzup_len);
13570 } else {
13571 vzup_code = pusha_code; // dummy
13572 }
13573
13574 assert(masm.code()->total_oop_size() == 0 &&
13575 masm.code()->total_metadata_size() == 0 &&
13576 masm.code()->total_relocation_size() == 0,
13577 "pre-computed code can't reference oops, metadata or contain relocations");
13578 }
13579
13580 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13581 assert(src != NULL, "code to copy must have been pre-computed");
13582 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13583 address end = code_section->end();
13584 memcpy(end, src, src_len);
13585 code_section->set_end(end + src_len);
13586 }
13587
13588 void Assembler::popa() { // 64bit
13589 emit_copy(code_section(), popa_code, popa_len);
13590 }
13591
13592 void Assembler::popa_uncached() { // 64bit
13593 movq(r15, Address(rsp, 0));
13594 movq(r14, Address(rsp, wordSize));
13595 movq(r13, Address(rsp, 2 * wordSize));
13596 movq(r12, Address(rsp, 3 * wordSize));
13597 movq(r11, Address(rsp, 4 * wordSize));
13598 movq(r10, Address(rsp, 5 * wordSize));
13599 movq(r9, Address(rsp, 6 * wordSize));
13600 movq(r8, Address(rsp, 7 * wordSize));
13601 movq(rdi, Address(rsp, 8 * wordSize));
13602 movq(rsi, Address(rsp, 9 * wordSize));
13603 movq(rbp, Address(rsp, 10 * wordSize));
13604 // Skip rsp as it is restored automatically to the value
13605 // before the corresponding pusha when popa is done.
13606 movq(rbx, Address(rsp, 12 * wordSize));
13607 movq(rdx, Address(rsp, 13 * wordSize));
13608 movq(rcx, Address(rsp, 14 * wordSize));
13609 movq(rax, Address(rsp, 15 * wordSize));
13610
13611 addq(rsp, 16 * wordSize);
13612 }
13613
13614 // Does not actually store the value of rsp on the stack.
13615 // The slot for rsp just contains an arbitrary value.
13616 void Assembler::pusha() { // 64bit
13617 emit_copy(code_section(), pusha_code, pusha_len);
13618 }
13619
13620 // Does not actually store the value of rsp on the stack.
13621 // The slot for rsp just contains an arbitrary value.
13622 void Assembler::pusha_uncached() { // 64bit
13623 subq(rsp, 16 * wordSize);
13624
13625 movq(Address(rsp, 15 * wordSize), rax);
13626 movq(Address(rsp, 14 * wordSize), rcx);
13627 movq(Address(rsp, 13 * wordSize), rdx);
13628 movq(Address(rsp, 12 * wordSize), rbx);
13629 // Skip rsp as the value is normally not used. There are a few places where
13630 // the original value of rsp needs to be known but that can be computed
13631 // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13632 movq(Address(rsp, 10 * wordSize), rbp);
13633 movq(Address(rsp, 9 * wordSize), rsi);
13634 movq(Address(rsp, 8 * wordSize), rdi);
13635 movq(Address(rsp, 7 * wordSize), r8);
13636 movq(Address(rsp, 6 * wordSize), r9);
13637 movq(Address(rsp, 5 * wordSize), r10);
13638 movq(Address(rsp, 4 * wordSize), r11);
13639 movq(Address(rsp, 3 * wordSize), r12);
13640 movq(Address(rsp, 2 * wordSize), r13);
13641 movq(Address(rsp, wordSize), r14);
13642 movq(Address(rsp, 0), r15);
13643 }
13644
13645 void Assembler::vzeroupper() {
13646 emit_copy(code_section(), vzup_code, vzup_len);
13647 }
13648
13649 void Assembler::vzeroall() {
13650 assert(VM_Version::supports_avx(), "requires AVX");
13651 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13652 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
13653 emit_int8(0x77);
13654 }
13655
13656 void Assembler::pushq(Address src) {
13657 InstructionMark im(this);
13658 emit_int16(get_prefixq(src), (unsigned char)0xFF);
13659 emit_operand(rsi, src, 0);
13660 }
13661
13662 void Assembler::rclq(Register dst, int imm8) {
13663 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13664 int encode = prefixq_and_encode(dst->encoding());
13665 if (imm8 == 1) {
13666 emit_int16((unsigned char)0xD1, (0xD0 | encode));
13667 } else {
13668 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13669 }
13670 }
13671
13672 void Assembler::rcrq(Register dst, int imm8) {
13673 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13674 int encode = prefixq_and_encode(dst->encoding());
13675 if (imm8 == 1) {
13676 emit_int16((unsigned char)0xD1, (0xD8 | encode));
13677 } else {
13678 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13679 }
13680 }
13681
13682 void Assembler::rorxl(Register dst, Register src, int imm8) {
13683 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13684 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13685 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13686 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13687 }
13688
13689 void Assembler::rorxl(Register dst, Address src, int imm8) {
13690 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13691 InstructionMark im(this);
13692 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13693 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13694 emit_int8((unsigned char)0xF0);
13695 emit_operand(dst, src, 1);
13696 emit_int8(imm8);
13697 }
13698
13699 void Assembler::rorxq(Register dst, Register src, int imm8) {
13700 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13701 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13702 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13703 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13704 }
13705
13706 void Assembler::rorxq(Register dst, Address src, int imm8) {
13707 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13708 InstructionMark im(this);
13709 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13710 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13711 emit_int8((unsigned char)0xF0);
13712 emit_operand(dst, src, 1);
13713 emit_int8(imm8);
13714 }
13715
13716 #ifdef _LP64
13717 void Assembler::salq(Address dst, int imm8) {
13718 InstructionMark im(this);
13719 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13720 if (imm8 == 1) {
13721 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13722 emit_operand(as_Register(4), dst, 0);
13723 }
13724 else {
13725 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13726 emit_operand(as_Register(4), dst, 1);
13727 emit_int8(imm8);
13728 }
13729 }
13730
13731 void Assembler::salq(Address dst) {
13732 InstructionMark im(this);
13733 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13734 emit_operand(as_Register(4), dst, 0);
13735 }
13736
13737 void Assembler::salq(Register dst, int imm8) {
13738 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13739 int encode = prefixq_and_encode(dst->encoding());
13740 if (imm8 == 1) {
13741 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13742 } else {
13743 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13744 }
13745 }
13746
13747 void Assembler::salq(Register dst) {
13748 int encode = prefixq_and_encode(dst->encoding());
13749 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13750 }
13751
13752 void Assembler::sarq(Address dst, int imm8) {
13753 InstructionMark im(this);
13754 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13755 if (imm8 == 1) {
13756 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13757 emit_operand(as_Register(7), dst, 0);
13758 }
13759 else {
13760 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13761 emit_operand(as_Register(7), dst, 1);
13762 emit_int8(imm8);
13763 }
13764 }
13765
13766 void Assembler::sarq(Address dst) {
13767 InstructionMark im(this);
13768 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13769 emit_operand(as_Register(7), dst, 0);
13770 }
13771
13772 void Assembler::sarq(Register dst, int imm8) {
13773 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13774 int encode = prefixq_and_encode(dst->encoding());
13775 if (imm8 == 1) {
13776 emit_int16((unsigned char)0xD1, (0xF8 | encode));
13777 } else {
13778 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13779 }
13780 }
13781
13782 void Assembler::sarq(Register dst) {
13783 int encode = prefixq_and_encode(dst->encoding());
13784 emit_int16((unsigned char)0xD3, (0xF8 | encode));
13785 }
13786 #endif
13787
13788 void Assembler::sbbq(Address dst, int32_t imm32) {
13789 InstructionMark im(this);
13790 prefixq(dst);
13791 emit_arith_operand(0x81, rbx, dst, imm32);
13792 }
13793
13794 void Assembler::sbbq(Register dst, int32_t imm32) {
13795 (void) prefixq_and_encode(dst->encoding());
13796 emit_arith(0x81, 0xD8, dst, imm32);
13797 }
13798
13799 void Assembler::sbbq(Register dst, Address src) {
13800 InstructionMark im(this);
13801 emit_int16(get_prefixq(src, dst), 0x1B);
13802 emit_operand(dst, src, 0);
13803 }
13804
13805 void Assembler::sbbq(Register dst, Register src) {
13806 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13807 emit_arith(0x1B, 0xC0, dst, src);
13808 }
13809
13810 void Assembler::shlq(Register dst, int imm8) {
13811 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13812 int encode = prefixq_and_encode(dst->encoding());
13813 if (imm8 == 1) {
13814 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13815 } else {
13816 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13817 }
13818 }
13819
13820 void Assembler::shlq(Register dst) {
13821 int encode = prefixq_and_encode(dst->encoding());
13822 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13823 }
13824
13825 void Assembler::shrq(Register dst, int imm8) {
13826 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13827 int encode = prefixq_and_encode(dst->encoding());
13828 if (imm8 == 1) {
13829 emit_int16((unsigned char)0xD1, (0xE8 | encode));
13830 }
13831 else {
13832 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13833 }
13834 }
13835
13836 void Assembler::shrq(Register dst) {
13837 int encode = prefixq_and_encode(dst->encoding());
13838 emit_int16((unsigned char)0xD3, 0xE8 | encode);
13839 }
13840
13841 void Assembler::shrq(Address dst) {
13842 InstructionMark im(this);
13843 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13844 emit_operand(as_Register(5), dst, 0);
13845 }
13846
13847 void Assembler::shrq(Address dst, int imm8) {
13848 InstructionMark im(this);
13849 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13850 if (imm8 == 1) {
13851 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13852 emit_operand(as_Register(5), dst, 0);
13853 }
13854 else {
13855 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13856 emit_operand(as_Register(5), dst, 1);
13857 emit_int8(imm8);
13858 }
13859 }
13860
13861 void Assembler::subq(Address dst, int32_t imm32) {
13862 InstructionMark im(this);
13863 prefixq(dst);
13864 emit_arith_operand(0x81, rbp, dst, imm32);
13865 }
13866
13867 void Assembler::subq(Address dst, Register src) {
13868 InstructionMark im(this);
13869 emit_int16(get_prefixq(dst, src), 0x29);
13870 emit_operand(src, dst, 0);
13871 }
13872
13873 void Assembler::subq(Register dst, int32_t imm32) {
13874 (void) prefixq_and_encode(dst->encoding());
13875 emit_arith(0x81, 0xE8, dst, imm32);
13876 }
13877
13878 // Force generation of a 4 byte immediate value even if it fits into 8bit
13879 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13880 (void) prefixq_and_encode(dst->encoding());
13881 emit_arith_imm32(0x81, 0xE8, dst, imm32);
13882 }
13883
13884 void Assembler::subq(Register dst, Address src) {
13885 InstructionMark im(this);
13886 emit_int16(get_prefixq(src, dst), 0x2B);
13887 emit_operand(dst, src, 0);
13888 }
13889
13890 void Assembler::subq(Register dst, Register src) {
13891 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13892 emit_arith(0x2B, 0xC0, dst, src);
13893 }
13894
13895 void Assembler::testq(Address dst, int32_t imm32) {
13896 InstructionMark im(this);
13897 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13898 emit_operand(as_Register(0), dst, 4);
13899 emit_int32(imm32);
13900 }
13901
13902 void Assembler::testq(Register dst, int32_t imm32) {
13903 // not using emit_arith because test
13904 // doesn't support sign-extension of
13905 // 8bit operands
13906 if (dst == rax) {
13907 prefix(REX_W);
13908 emit_int8((unsigned char)0xA9);
13909 emit_int32(imm32);
13910 } else {
13911 int encode = dst->encoding();
13912 encode = prefixq_and_encode(encode);
13913 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13914 emit_int32(imm32);
13915 }
13916 }
13917
13918 void Assembler::testq(Register dst, Register src) {
13919 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13920 emit_arith(0x85, 0xC0, dst, src);
13921 }
13922
13923 void Assembler::testq(Register dst, Address src) {
13924 InstructionMark im(this);
13925 emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13926 emit_operand(dst, src, 0);
13927 }
13928
13929 void Assembler::xaddq(Address dst, Register src) {
13930 InstructionMark im(this);
13931 emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13932 emit_operand(src, dst, 0);
13933 }
13934
13935 void Assembler::xchgq(Register dst, Address src) {
13936 InstructionMark im(this);
13937 emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13938 emit_operand(dst, src, 0);
13939 }
13940
13941 void Assembler::xchgq(Register dst, Register src) {
13942 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13943 emit_int16((unsigned char)0x87, (0xc0 | encode));
13944 }
13945
13946 void Assembler::xorq(Register dst, Register src) {
13947 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13948 emit_arith(0x33, 0xC0, dst, src);
13949 }
13950
13951 void Assembler::xorq(Register dst, Address src) {
13952 InstructionMark im(this);
13953 emit_int16(get_prefixq(src, dst), 0x33);
13954 emit_operand(dst, src, 0);
13955 }
13956
13957 void Assembler::xorq(Register dst, int32_t imm32) {
13958 (void) prefixq_and_encode(dst->encoding());
13959 emit_arith(0x81, 0xF0, dst, imm32);
13960 }
13961
13962 void Assembler::xorq(Address dst, int32_t imm32) {
13963 InstructionMark im(this);
13964 prefixq(dst);
13965 emit_arith_operand(0x81, as_Register(6), dst, imm32);
13966 }
13967
13968 void Assembler::xorq(Address dst, Register src) {
13969 InstructionMark im(this);
13970 emit_int16(get_prefixq(dst, src), 0x31);
13971 emit_operand(src, dst, 0);
13972 }
13973
13974 #endif // !LP64
13975
13976 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13977 if (VM_Version::supports_evex()) {
13978 _tuple_type = tuple_type;
13979 _input_size_in_bits = input_size_in_bits;
13980 }
13981 }