1 /*
2 * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableBarrierSet.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/objectMonitor.hpp"
34 #include "runtime/os.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37 #include "runtime/vm_version.hpp"
38 #include "utilities/macros.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
49 // Implementation of AddressLiteral
50
51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
53 // -----------------Table 4.5 -------------------- //
54 16, 32, 64, // EVEX_FV(0)
55 4, 4, 4, // EVEX_FV(1) - with Evex.b
56 16, 32, 64, // EVEX_FV(2) - with Evex.w
57 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
58 8, 16, 32, // EVEX_HV(0)
59 4, 4, 4, // EVEX_HV(1) - with Evex.b
60 // -----------------Table 4.6 -------------------- //
61 16, 32, 64, // EVEX_FVM(0)
62 1, 1, 1, // EVEX_T1S(0)
63 2, 2, 2, // EVEX_T1S(1)
64 4, 4, 4, // EVEX_T1S(2)
65 8, 8, 8, // EVEX_T1S(3)
66 4, 4, 4, // EVEX_T1F(0)
67 8, 8, 8, // EVEX_T1F(1)
68 8, 8, 8, // EVEX_T2(0)
69 0, 16, 16, // EVEX_T2(1)
70 0, 16, 16, // EVEX_T4(0)
71 0, 0, 32, // EVEX_T4(1)
72 0, 0, 32, // EVEX_T8(0)
73 8, 16, 32, // EVEX_HVM(0)
74 4, 8, 16, // EVEX_QVM(0)
75 2, 4, 8, // EVEX_OVM(0)
76 16, 16, 16, // EVEX_M128(0)
77 8, 32, 64, // EVEX_DUP(0)
78 0, 0, 0 // EVEX_NTUP
79 };
80
81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
82 _is_lval = false;
83 _target = target;
84 switch (rtype) {
85 case relocInfo::oop_type:
86 case relocInfo::metadata_type:
87 // Oops are a special case. Normally they would be their own section
88 // but in cases like icBuffer they are literals in the code stream that
89 // we don't have a section for. We use none so that we get a literal address
90 // which is always patchable.
91 break;
92 case relocInfo::external_word_type:
93 _rspec = external_word_Relocation::spec(target);
94 break;
95 case relocInfo::internal_word_type:
96 _rspec = internal_word_Relocation::spec(target);
97 break;
98 case relocInfo::opt_virtual_call_type:
99 _rspec = opt_virtual_call_Relocation::spec();
100 break;
101 case relocInfo::static_call_type:
102 _rspec = static_call_Relocation::spec();
103 break;
104 case relocInfo::runtime_call_type:
105 _rspec = runtime_call_Relocation::spec();
106 break;
107 case relocInfo::poll_type:
108 case relocInfo::poll_return_type:
109 _rspec = Relocation::spec_simple(rtype);
110 break;
111 case relocInfo::none:
112 break;
113 default:
114 ShouldNotReachHere();
115 break;
116 }
117 }
118
119 // Implementation of Address
120
121 #ifdef _LP64
122
123 Address Address::make_array(ArrayAddress adr) {
124 // Not implementable on 64bit machines
125 // Should have been handled higher up the call chain.
126 ShouldNotReachHere();
127 return Address();
128 }
129
130 // exceedingly dangerous constructor
131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
132 _base = noreg;
133 _index = noreg;
134 _scale = no_scale;
135 _disp = disp;
136 _xmmindex = xnoreg;
137 _isxmmindex = false;
138 switch (rtype) {
139 case relocInfo::external_word_type:
140 _rspec = external_word_Relocation::spec(loc);
141 break;
142 case relocInfo::internal_word_type:
143 _rspec = internal_word_Relocation::spec(loc);
144 break;
145 case relocInfo::runtime_call_type:
146 // HMM
147 _rspec = runtime_call_Relocation::spec();
148 break;
149 case relocInfo::poll_type:
150 case relocInfo::poll_return_type:
151 _rspec = Relocation::spec_simple(rtype);
152 break;
153 case relocInfo::none:
154 break;
155 default:
156 ShouldNotReachHere();
157 }
158 }
159 #else // LP64
160
161 Address Address::make_array(ArrayAddress adr) {
162 AddressLiteral base = adr.base();
163 Address index = adr.index();
164 assert(index._disp == 0, "must not have disp"); // maybe it can?
165 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
166 array._rspec = base._rspec;
167 return array;
168 }
169
170 // exceedingly dangerous constructor
171 Address::Address(address loc, RelocationHolder spec) {
172 _base = noreg;
173 _index = noreg;
174 _scale = no_scale;
175 _disp = (intptr_t) loc;
176 _rspec = spec;
177 _xmmindex = xnoreg;
178 _isxmmindex = false;
179 }
180
181 #endif // _LP64
182
183
184
185 // Convert the raw encoding form into the form expected by the constructor for
186 // Address. An index of 4 (rsp) corresponds to having no index, so convert
187 // that to noreg for the Address constructor.
188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
189 RelocationHolder rspec = RelocationHolder::none;
190 if (disp_reloc != relocInfo::none) {
191 rspec = Relocation::spec_simple(disp_reloc);
192 }
193 bool valid_index = index != rsp->encoding();
194 if (valid_index) {
195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
196 madr._rspec = rspec;
197 return madr;
198 } else {
199 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
200 madr._rspec = rspec;
201 return madr;
202 }
203 }
204
205 // Implementation of Assembler
206
207 int AbstractAssembler::code_fill_byte() {
208 return (u_char)'\xF4'; // hlt
209 }
210
211 void Assembler::init_attributes(void) {
212 _legacy_mode_bw = (VM_Version::supports_avx512bw() == false);
213 _legacy_mode_dq = (VM_Version::supports_avx512dq() == false);
214 _legacy_mode_vl = (VM_Version::supports_avx512vl() == false);
215 _legacy_mode_vlbw = (VM_Version::supports_avx512vlbw() == false);
216 NOT_LP64(_is_managed = false;)
217 _attributes = NULL;
218 }
219
220
221 void Assembler::membar(Membar_mask_bits order_constraint) {
222 // We only have to handle StoreLoad
223 if (order_constraint & StoreLoad) {
224 // All usable chips support "locked" instructions which suffice
225 // as barriers, and are much faster than the alternative of
226 // using cpuid instruction. We use here a locked add [esp-C],0.
227 // This is conveniently otherwise a no-op except for blowing
228 // flags, and introducing a false dependency on target memory
229 // location. We can't do anything with flags, but we can avoid
230 // memory dependencies in the current method by locked-adding
231 // somewhere else on the stack. Doing [esp+C] will collide with
232 // something on stack in current method, hence we go for [esp-C].
233 // It is convenient since it is almost always in data cache, for
234 // any small C. We need to step back from SP to avoid data
235 // dependencies with other things on below SP (callee-saves, for
236 // example). Without a clear way to figure out the minimal safe
237 // distance from SP, it makes sense to step back the complete
238 // cache line, as this will also avoid possible second-order effects
239 // with locked ops against the cache line. Our choice of offset
240 // is bounded by x86 operand encoding, which should stay within
241 // [-128; +127] to have the 8-byte displacement encoding.
242 //
243 // Any change to this code may need to revisit other places in
244 // the code where this idiom is used, in particular the
245 // orderAccess code.
246
247 int offset = -VM_Version::L1_line_size();
248 if (offset < -128) {
249 offset = -128;
250 }
251
252 lock();
253 addl(Address(rsp, offset), 0);// Assert the lock# signal here
254 }
255 }
256
257 // make this go away someday
258 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
259 if (rtype == relocInfo::none)
260 emit_int32(data);
261 else
262 emit_data(data, Relocation::spec_simple(rtype), format);
263 }
264
265 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
266 assert(imm_operand == 0, "default format must be immediate in this file");
267 assert(inst_mark() != NULL, "must be inside InstructionMark");
268 if (rspec.type() != relocInfo::none) {
269 #ifdef ASSERT
270 check_relocation(rspec, format);
271 #endif
272 // Do not use AbstractAssembler::relocate, which is not intended for
273 // embedded words. Instead, relocate to the enclosing instruction.
274
275 // hack. call32 is too wide for mask so use disp32
276 if (format == call32_operand)
277 code_section()->relocate(inst_mark(), rspec, disp32_operand);
278 else
279 code_section()->relocate(inst_mark(), rspec, format);
280 }
281 emit_int32(data);
282 }
283
284 static int encode(Register r) {
285 int enc = r->encoding();
286 if (enc >= 8) {
287 enc -= 8;
288 }
289 return enc;
290 }
291
292 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
293 assert(dst->has_byte_register(), "must have byte register");
294 assert(isByte(op1) && isByte(op2), "wrong opcode");
295 assert(isByte(imm8), "not a byte");
296 assert((op1 & 0x01) == 0, "should be 8bit operation");
297 emit_int24(op1, (op2 | encode(dst)), imm8);
298 }
299
300
301 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
302 assert(isByte(op1) && isByte(op2), "wrong opcode");
303 assert(op1 == 0x81, "Unexpected opcode");
304 if (is8bit(imm32)) {
305 emit_int24(op1 | 0x02, // set sign bit
306 op2 | encode(dst),
307 imm32 & 0xFF);
308 } else if (dst == rax) {
309 switch (op2) {
310 case 0xD0: emit_int8(0x15); break; // adc
311 case 0xC0: emit_int8(0x05); break; // add
312 case 0xE0: emit_int8(0x25); break; // and
313 case 0xF8: emit_int8(0x3D); break; // cmp
314 case 0xC8: emit_int8(0x0D); break; // or
315 case 0xD8: emit_int8(0x1D); break; // sbb
316 case 0xE8: emit_int8(0x2D); break; // sub
317 case 0xF0: emit_int8(0x35); break; // xor
318 default: ShouldNotReachHere();
319 }
320 emit_int32(imm32);
321 } else {
322 emit_int16(op1, (op2 | encode(dst)));
323 emit_int32(imm32);
324 }
325 }
326
327 // Force generation of a 4 byte immediate value even if it fits into 8bit
328 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
329 assert(isByte(op1) && isByte(op2), "wrong opcode");
330 assert((op1 & 0x01) == 1, "should be 32bit operation");
331 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
332 emit_int16(op1, (op2 | encode(dst)));
333 emit_int32(imm32);
334 }
335
336 // immediate-to-memory forms
337 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
338 assert((op1 & 0x01) == 1, "should be 32bit operation");
339 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
340 if (is8bit(imm32)) {
341 emit_int8(op1 | 0x02); // set sign bit
342 emit_operand(rm, adr, 1);
343 emit_int8(imm32 & 0xFF);
344 } else {
345 emit_int8(op1);
346 emit_operand(rm, adr, 4);
347 emit_int32(imm32);
348 }
349 }
350
351
352 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
353 assert(isByte(op1) && isByte(op2), "wrong opcode");
354 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
355 }
356
357
358 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
359 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
360 int mod_idx = 0;
361 // We will test if the displacement fits the compressed format and if so
362 // apply the compression to the displacement iff the result is8bit.
363 if (VM_Version::supports_evex() && is_evex_inst) {
364 switch (cur_tuple_type) {
365 case EVEX_FV:
366 if ((cur_encoding & VEX_W) == VEX_W) {
367 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
368 } else {
369 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
370 }
371 break;
372
373 case EVEX_HV:
374 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
375 break;
376
377 case EVEX_FVM:
378 break;
379
380 case EVEX_T1S:
381 switch (in_size_in_bits) {
382 case EVEX_8bit:
383 break;
384
385 case EVEX_16bit:
386 mod_idx = 1;
387 break;
388
389 case EVEX_32bit:
390 mod_idx = 2;
391 break;
392
393 case EVEX_64bit:
394 mod_idx = 3;
395 break;
396 }
397 break;
398
399 case EVEX_T1F:
400 case EVEX_T2:
401 case EVEX_T4:
402 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
403 break;
404
405 case EVEX_T8:
406 break;
407
408 case EVEX_HVM:
409 break;
410
411 case EVEX_QVM:
412 break;
413
414 case EVEX_OVM:
415 break;
416
417 case EVEX_M128:
418 break;
419
420 case EVEX_DUP:
421 break;
422
423 default:
424 assert(0, "no valid evex tuple_table entry");
425 break;
426 }
427
428 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
429 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
430 if ((disp % disp_factor) == 0) {
431 int new_disp = disp / disp_factor;
432 if ((-0x80 <= new_disp && new_disp < 0x80)) {
433 disp = new_disp;
434 }
435 } else {
436 return false;
437 }
438 }
439 }
440 return (-0x80 <= disp && disp < 0x80);
441 }
442
443
444 bool Assembler::emit_compressed_disp_byte(int &disp) {
445 int mod_idx = 0;
446 // We will test if the displacement fits the compressed format and if so
447 // apply the compression to the displacement iff the result is8bit.
448 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
449 int evex_encoding = _attributes->get_evex_encoding();
450 int tuple_type = _attributes->get_tuple_type();
451 switch (tuple_type) {
452 case EVEX_FV:
453 if ((evex_encoding & VEX_W) == VEX_W) {
454 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
455 } else {
456 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
457 }
458 break;
459
460 case EVEX_HV:
461 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
462 break;
463
464 case EVEX_FVM:
465 break;
466
467 case EVEX_T1S:
468 switch (_attributes->get_input_size()) {
469 case EVEX_8bit:
470 break;
471
472 case EVEX_16bit:
473 mod_idx = 1;
474 break;
475
476 case EVEX_32bit:
477 mod_idx = 2;
478 break;
479
480 case EVEX_64bit:
481 mod_idx = 3;
482 break;
483 }
484 break;
485
486 case EVEX_T1F:
487 case EVEX_T2:
488 case EVEX_T4:
489 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
490 break;
491
492 case EVEX_T8:
493 break;
494
495 case EVEX_HVM:
496 break;
497
498 case EVEX_QVM:
499 break;
500
501 case EVEX_OVM:
502 break;
503
504 case EVEX_M128:
505 break;
506
507 case EVEX_DUP:
508 break;
509
510 default:
511 assert(0, "no valid evex tuple_table entry");
512 break;
513 }
514
515 int vector_len = _attributes->get_vector_len();
516 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
517 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
518 if ((disp % disp_factor) == 0) {
519 int new_disp = disp / disp_factor;
520 if (is8bit(new_disp)) {
521 disp = new_disp;
522 }
523 } else {
524 return false;
525 }
526 }
527 }
528 return is8bit(disp);
529 }
530
531 static bool is_valid_encoding(int reg_enc) {
532 return reg_enc >= 0;
533 }
534
535 static int raw_encode(Register reg) {
536 assert(reg == noreg || reg->is_valid(), "sanity");
537 int reg_enc = (intptr_t)reg;
538 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
539 return reg_enc;
540 }
541
542 static int raw_encode(XMMRegister xmmreg) {
543 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
544 int xmmreg_enc = (intptr_t)xmmreg;
545 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
546 return xmmreg_enc;
547 }
548
549 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
550 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
551 }
552
553 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
554 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
555 }
556
557 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
558 assert((mod & 3) != 0b11, "forbidden");
559 int modrm = modrm_encoding(mod, dst_enc, src_enc);
560 emit_int8(modrm);
561 }
562
563 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
564 int disp) {
565 int modrm = modrm_encoding(mod, dst_enc, src_enc);
566 emit_int16(modrm, disp & 0xFF);
567 }
568
569 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
570 Address::ScaleFactor scale, int index_enc, int base_enc) {
571 int modrm = modrm_encoding(mod, dst_enc, src_enc);
572 int sib = sib_encoding(scale, index_enc, base_enc);
573 emit_int16(modrm, sib);
574 }
575
576 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
577 Address::ScaleFactor scale, int index_enc, int base_enc,
578 int disp) {
579 int modrm = modrm_encoding(mod, dst_enc, src_enc);
580 int sib = sib_encoding(scale, index_enc, base_enc);
581 emit_int24(modrm, sib, disp & 0xFF);
582 }
583
584 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
585 Address::ScaleFactor scale, int disp,
586 RelocationHolder const& rspec,
587 int rip_relative_correction) {
588 bool no_relocation = (rspec.type() == relocInfo::none);
589
590 if (is_valid_encoding(base_enc)) {
591 if (is_valid_encoding(index_enc)) {
592 assert(scale != Address::no_scale, "inconsistent address");
593 // [base + index*scale + disp]
594 if (disp == 0 && no_relocation &&
595 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
596 // [base + index*scale]
597 // [00 reg 100][ss index base]
598 emit_modrm_sib(0b00, reg_enc, 0b100,
599 scale, index_enc, base_enc);
600 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
601 // [base + index*scale + imm8]
602 // [01 reg 100][ss index base] imm8
603 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
604 scale, index_enc, base_enc,
605 disp);
606 } else {
607 // [base + index*scale + disp32]
608 // [10 reg 100][ss index base] disp32
609 emit_modrm_sib(0b10, reg_enc, 0b100,
610 scale, index_enc, base_enc);
611 emit_data(disp, rspec, disp32_operand);
612 }
613 } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
614 // [rsp + disp]
615 if (disp == 0 && no_relocation) {
616 // [rsp]
617 // [00 reg 100][00 100 100]
618 emit_modrm_sib(0b00, reg_enc, 0b100,
619 Address::times_1, 0b100, 0b100);
620 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
621 // [rsp + imm8]
622 // [01 reg 100][00 100 100] disp8
623 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
624 Address::times_1, 0b100, 0b100,
625 disp);
626 } else {
627 // [rsp + imm32]
628 // [10 reg 100][00 100 100] disp32
629 emit_modrm_sib(0b10, reg_enc, 0b100,
630 Address::times_1, 0b100, 0b100);
631 emit_data(disp, rspec, disp32_operand);
632 }
633 } else {
634 // [base + disp]
635 assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
636 if (disp == 0 && no_relocation &&
637 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
638 // [base]
639 // [00 reg base]
640 emit_modrm(0, reg_enc, base_enc);
641 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
642 // [base + disp8]
643 // [01 reg base] disp8
644 emit_modrm_disp8(0b01, reg_enc, base_enc,
645 disp);
646 } else {
647 // [base + disp32]
648 // [10 reg base] disp32
649 emit_modrm(0b10, reg_enc, base_enc);
650 emit_data(disp, rspec, disp32_operand);
651 }
652 }
653 } else {
654 if (is_valid_encoding(index_enc)) {
655 assert(scale != Address::no_scale, "inconsistent address");
656 // base == noreg
657 // [index*scale + disp]
658 // [00 reg 100][ss index 101] disp32
659 emit_modrm_sib(0b00, reg_enc, 0b100,
660 scale, index_enc, 0b101 /* no base */);
661 emit_data(disp, rspec, disp32_operand);
662 } else if (!no_relocation) {
663 // base == noreg, index == noreg
664 // [disp] (64bit) RIP-RELATIVE (32bit) abs
665 // [00 reg 101] disp32
666
667 emit_modrm(0b00, reg_enc, 0b101 /* no base */);
668 // Note that the RIP-rel. correction applies to the generated
669 // disp field, but _not_ to the target address in the rspec.
670
671 // disp was created by converting the target address minus the pc
672 // at the start of the instruction. That needs more correction here.
673 // intptr_t disp = target - next_ip;
674 assert(inst_mark() != NULL, "must be inside InstructionMark");
675 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
676 int64_t adjusted = disp;
677 // Do rip-rel adjustment for 64bit
678 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
679 assert(is_simm32(adjusted),
680 "must be 32bit offset (RIP relative address)");
681 emit_data((int32_t) adjusted, rspec, disp32_operand);
682
683 } else {
684 // base == noreg, index == noreg, no_relocation == true
685 // 32bit never did this, did everything as the rip-rel/disp code above
686 // [disp] ABSOLUTE
687 // [00 reg 100][00 100 101] disp32
688 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
689 Address::times_1, 0b100, 0b101);
690 emit_data(disp, rspec, disp32_operand);
691 }
692 }
693 }
694
695 void Assembler::emit_operand(Register reg, Register base, Register index,
696 Address::ScaleFactor scale, int disp,
697 RelocationHolder const& rspec,
698 int rip_relative_correction) {
699 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
700 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
701 scale, disp, rspec, rip_relative_correction);
702
703 }
704 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
705 Address::ScaleFactor scale, int disp,
706 RelocationHolder const& rspec) {
707 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
708 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
709 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
710 scale, disp, rspec);
711 }
712
713 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
714 Address::ScaleFactor scale, int disp,
715 RelocationHolder const& rspec) {
716 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
717 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
718 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
719 scale, disp, rspec, /* rip_relative_correction */ 0);
720 }
721
722 // Secret local extension to Assembler::WhichOperand:
723 #define end_pc_operand (_WhichOperand_limit)
724
725 address Assembler::locate_operand(address inst, WhichOperand which) {
726 // Decode the given instruction, and return the address of
727 // an embedded 32-bit operand word.
728
729 // If "which" is disp32_operand, selects the displacement portion
730 // of an effective address specifier.
731 // If "which" is imm64_operand, selects the trailing immediate constant.
732 // If "which" is call32_operand, selects the displacement of a call or jump.
733 // Caller is responsible for ensuring that there is such an operand,
734 // and that it is 32/64 bits wide.
735
736 // If "which" is end_pc_operand, find the end of the instruction.
737
738 address ip = inst;
739 bool is_64bit = false;
740
741 debug_only(bool has_disp32 = false);
742 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
743
744 again_after_prefix:
745 switch (0xFF & *ip++) {
746
747 // These convenience macros generate groups of "case" labels for the switch.
748 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
749 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
750 case (x)+4: case (x)+5: case (x)+6: case (x)+7
751 #define REP16(x) REP8((x)+0): \
752 case REP8((x)+8)
753
754 case CS_segment:
755 case SS_segment:
756 case DS_segment:
757 case ES_segment:
758 case FS_segment:
759 case GS_segment:
760 // Seems dubious
761 LP64_ONLY(assert(false, "shouldn't have that prefix"));
762 assert(ip == inst+1, "only one prefix allowed");
763 goto again_after_prefix;
764
765 case 0x67:
766 case REX:
767 case REX_B:
768 case REX_X:
769 case REX_XB:
770 case REX_R:
771 case REX_RB:
772 case REX_RX:
773 case REX_RXB:
774 NOT_LP64(assert(false, "64bit prefixes"));
775 goto again_after_prefix;
776
777 case REX_W:
778 case REX_WB:
779 case REX_WX:
780 case REX_WXB:
781 case REX_WR:
782 case REX_WRB:
783 case REX_WRX:
784 case REX_WRXB:
785 NOT_LP64(assert(false, "64bit prefixes"));
786 is_64bit = true;
787 goto again_after_prefix;
788
789 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
790 case 0x88: // movb a, r
791 case 0x89: // movl a, r
792 case 0x8A: // movb r, a
793 case 0x8B: // movl r, a
794 case 0x8F: // popl a
795 debug_only(has_disp32 = true);
796 break;
797
798 case 0x68: // pushq #32
799 if (which == end_pc_operand) {
800 return ip + 4;
801 }
802 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
803 return ip; // not produced by emit_operand
804
805 case 0x66: // movw ... (size prefix)
806 again_after_size_prefix2:
807 switch (0xFF & *ip++) {
808 case REX:
809 case REX_B:
810 case REX_X:
811 case REX_XB:
812 case REX_R:
813 case REX_RB:
814 case REX_RX:
815 case REX_RXB:
816 case REX_W:
817 case REX_WB:
818 case REX_WX:
819 case REX_WXB:
820 case REX_WR:
821 case REX_WRB:
822 case REX_WRX:
823 case REX_WRXB:
824 NOT_LP64(assert(false, "64bit prefix found"));
825 goto again_after_size_prefix2;
826 case 0x8B: // movw r, a
827 case 0x89: // movw a, r
828 debug_only(has_disp32 = true);
829 break;
830 case 0xC7: // movw a, #16
831 debug_only(has_disp32 = true);
832 tail_size = 2; // the imm16
833 break;
834 case 0x0F: // several SSE/SSE2 variants
835 ip--; // reparse the 0x0F
836 goto again_after_prefix;
837 default:
838 ShouldNotReachHere();
839 }
840 break;
841
842 case REP8(0xB8): // movl/q r, #32/#64(oop?)
843 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
844 // these asserts are somewhat nonsensical
845 #ifndef _LP64
846 assert(which == imm_operand || which == disp32_operand,
847 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
848 #else
849 assert((which == call32_operand || which == imm_operand) && is_64bit ||
850 which == narrow_oop_operand && !is_64bit,
851 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
852 #endif // _LP64
853 return ip;
854
855 case 0x69: // imul r, a, #32
856 case 0xC7: // movl a, #32(oop?)
857 tail_size = 4;
858 debug_only(has_disp32 = true); // has both kinds of operands!
859 break;
860
861 case 0x0F: // movx..., etc.
862 switch (0xFF & *ip++) {
863 case 0x3A: // pcmpestri
864 tail_size = 1;
865 case 0x38: // ptest, pmovzxbw
866 ip++; // skip opcode
867 debug_only(has_disp32 = true); // has both kinds of operands!
868 break;
869
870 case 0x70: // pshufd r, r/a, #8
871 debug_only(has_disp32 = true); // has both kinds of operands!
872 case 0x73: // psrldq r, #8
873 tail_size = 1;
874 break;
875
876 case 0x12: // movlps
877 case 0x28: // movaps
878 case 0x2E: // ucomiss
879 case 0x2F: // comiss
880 case 0x54: // andps
881 case 0x55: // andnps
882 case 0x56: // orps
883 case 0x57: // xorps
884 case 0x58: // addpd
885 case 0x59: // mulpd
886 case 0x6E: // movd
887 case 0x7E: // movd
888 case 0x6F: // movdq
889 case 0x7F: // movdq
890 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
891 case 0xFE: // paddd
892 debug_only(has_disp32 = true);
893 break;
894
895 case 0xAD: // shrd r, a, %cl
896 case 0xAF: // imul r, a
897 case 0xBE: // movsbl r, a (movsxb)
898 case 0xBF: // movswl r, a (movsxw)
899 case 0xB6: // movzbl r, a (movzxb)
900 case 0xB7: // movzwl r, a (movzxw)
901 case REP16(0x40): // cmovl cc, r, a
902 case 0xB0: // cmpxchgb
903 case 0xB1: // cmpxchg
904 case 0xC1: // xaddl
905 case 0xC7: // cmpxchg8
906 case REP16(0x90): // setcc a
907 debug_only(has_disp32 = true);
908 // fall out of the switch to decode the address
909 break;
910
911 case 0xC4: // pinsrw r, a, #8
912 debug_only(has_disp32 = true);
913 case 0xC5: // pextrw r, r, #8
914 tail_size = 1; // the imm8
915 break;
916
917 case 0xAC: // shrd r, a, #8
918 debug_only(has_disp32 = true);
919 tail_size = 1; // the imm8
920 break;
921
922 case REP16(0x80): // jcc rdisp32
923 if (which == end_pc_operand) return ip + 4;
924 assert(which == call32_operand, "jcc has no disp32 or imm");
925 return ip;
926 default:
927 ShouldNotReachHere();
928 }
929 break;
930
931 case 0x81: // addl a, #32; addl r, #32
932 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
933 // on 32bit in the case of cmpl, the imm might be an oop
934 tail_size = 4;
935 debug_only(has_disp32 = true); // has both kinds of operands!
936 break;
937
938 case 0x83: // addl a, #8; addl r, #8
939 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
940 debug_only(has_disp32 = true); // has both kinds of operands!
941 tail_size = 1;
942 break;
943
944 case 0x15: // adc rax, #32
945 case 0x05: // add rax, #32
946 case 0x25: // and rax, #32
947 case 0x3D: // cmp rax, #32
948 case 0x0D: // or rax, #32
949 case 0x1D: // sbb rax, #32
950 case 0x2D: // sub rax, #32
951 case 0x35: // xor rax, #32
952 return which == end_pc_operand ? ip + 4 : ip;
953
954 case 0x9B:
955 switch (0xFF & *ip++) {
956 case 0xD9: // fnstcw a
957 debug_only(has_disp32 = true);
958 break;
959 default:
960 ShouldNotReachHere();
961 }
962 break;
963
964 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
965 case REP4(0x10): // adc...
966 case REP4(0x20): // and...
967 case REP4(0x30): // xor...
968 case REP4(0x08): // or...
969 case REP4(0x18): // sbb...
970 case REP4(0x28): // sub...
971 case 0xF7: // mull a
972 case 0x8D: // lea r, a
973 case 0x87: // xchg r, a
974 case REP4(0x38): // cmp...
975 case 0x85: // test r, a
976 debug_only(has_disp32 = true); // has both kinds of operands!
977 break;
978
979 case 0xA8: // testb rax, #8
980 return which == end_pc_operand ? ip + 1 : ip;
981 case 0xA9: // testl/testq rax, #32
982 return which == end_pc_operand ? ip + 4 : ip;
983
984 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
985 case 0xC6: // movb a, #8
986 case 0x80: // cmpb a, #8
987 case 0x6B: // imul r, a, #8
988 debug_only(has_disp32 = true); // has both kinds of operands!
989 tail_size = 1; // the imm8
990 break;
991
992 case 0xC4: // VEX_3bytes
993 case 0xC5: // VEX_2bytes
994 assert((UseAVX > 0), "shouldn't have VEX prefix");
995 assert(ip == inst+1, "no prefixes allowed");
996 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
997 // but they have prefix 0x0F and processed when 0x0F processed above.
998 //
999 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
1000 // instructions (these instructions are not supported in 64-bit mode).
1001 // To distinguish them bits [7:6] are set in the VEX second byte since
1002 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
1003 // those VEX bits REX and vvvv bits are inverted.
1004 //
1005 // Fortunately C2 doesn't generate these instructions so we don't need
1006 // to check for them in product version.
1007
1008 // Check second byte
1009 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
1010
1011 int vex_opcode;
1012 // First byte
1013 if ((0xFF & *inst) == VEX_3bytes) {
1014 vex_opcode = VEX_OPCODE_MASK & *ip;
1015 ip++; // third byte
1016 is_64bit = ((VEX_W & *ip) == VEX_W);
1017 } else {
1018 vex_opcode = VEX_OPCODE_0F;
1019 }
1020 ip++; // opcode
1021 // To find the end of instruction (which == end_pc_operand).
1022 switch (vex_opcode) {
1023 case VEX_OPCODE_0F:
1024 switch (0xFF & *ip) {
1025 case 0x70: // pshufd r, r/a, #8
1026 case 0x71: // ps[rl|ra|ll]w r, #8
1027 case 0x72: // ps[rl|ra|ll]d r, #8
1028 case 0x73: // ps[rl|ra|ll]q r, #8
1029 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
1030 case 0xC4: // pinsrw r, r, r/a, #8
1031 case 0xC5: // pextrw r/a, r, #8
1032 case 0xC6: // shufp[s|d] r, r, r/a, #8
1033 tail_size = 1; // the imm8
1034 break;
1035 }
1036 break;
1037 case VEX_OPCODE_0F_3A:
1038 tail_size = 1;
1039 break;
1040 }
1041 ip++; // skip opcode
1042 debug_only(has_disp32 = true); // has both kinds of operands!
1043 break;
1044
1045 case 0x62: // EVEX_4bytes
1046 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
1047 assert(ip == inst+1, "no prefixes allowed");
1048 // no EVEX collisions, all instructions that have 0x62 opcodes
1049 // have EVEX versions and are subopcodes of 0x66
1050 ip++; // skip P0 and examine W in P1
1051 is_64bit = ((VEX_W & *ip) == VEX_W);
1052 ip++; // move to P2
1053 ip++; // skip P2, move to opcode
1054 // To find the end of instruction (which == end_pc_operand).
1055 switch (0xFF & *ip) {
1056 case 0x22: // pinsrd r, r/a, #8
1057 case 0x61: // pcmpestri r, r/a, #8
1058 case 0x70: // pshufd r, r/a, #8
1059 case 0x73: // psrldq r, #8
1060 case 0x1f: // evpcmpd/evpcmpq
1061 case 0x3f: // evpcmpb/evpcmpw
1062 tail_size = 1; // the imm8
1063 break;
1064 default:
1065 break;
1066 }
1067 ip++; // skip opcode
1068 debug_only(has_disp32 = true); // has both kinds of operands!
1069 break;
1070
1071 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
1072 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
1073 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
1074 case 0xDD: // fld_d a; fst_d a; fstp_d a
1075 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
1076 case 0xDF: // fild_d a; fistp_d a
1077 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
1078 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
1079 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
1080 debug_only(has_disp32 = true);
1081 break;
1082
1083 case 0xE8: // call rdisp32
1084 case 0xE9: // jmp rdisp32
1085 if (which == end_pc_operand) return ip + 4;
1086 assert(which == call32_operand, "call has no disp32 or imm");
1087 return ip;
1088
1089 case 0xF0: // Lock
1090 goto again_after_prefix;
1091
1092 case 0xF3: // For SSE
1093 case 0xF2: // For SSE2
1094 switch (0xFF & *ip++) {
1095 case REX:
1096 case REX_B:
1097 case REX_X:
1098 case REX_XB:
1099 case REX_R:
1100 case REX_RB:
1101 case REX_RX:
1102 case REX_RXB:
1103 case REX_W:
1104 case REX_WB:
1105 case REX_WX:
1106 case REX_WXB:
1107 case REX_WR:
1108 case REX_WRB:
1109 case REX_WRX:
1110 case REX_WRXB:
1111 NOT_LP64(assert(false, "found 64bit prefix"));
1112 ip++;
1113 default:
1114 ip++;
1115 }
1116 debug_only(has_disp32 = true); // has both kinds of operands!
1117 break;
1118
1119 default:
1120 ShouldNotReachHere();
1121
1122 #undef REP8
1123 #undef REP16
1124 }
1125
1126 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1127 #ifdef _LP64
1128 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1129 #else
1130 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1131 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1132 #endif // LP64
1133 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1134
1135 // parse the output of emit_operand
1136 int op2 = 0xFF & *ip++;
1137 int base = op2 & 0x07;
1138 int op3 = -1;
1139 const int b100 = 4;
1140 const int b101 = 5;
1141 if (base == b100 && (op2 >> 6) != 3) {
1142 op3 = 0xFF & *ip++;
1143 base = op3 & 0x07; // refetch the base
1144 }
1145 // now ip points at the disp (if any)
1146
1147 switch (op2 >> 6) {
1148 case 0:
1149 // [00 reg 100][ss index base]
1150 // [00 reg 100][00 100 esp]
1151 // [00 reg base]
1152 // [00 reg 100][ss index 101][disp32]
1153 // [00 reg 101] [disp32]
1154
1155 if (base == b101) {
1156 if (which == disp32_operand)
1157 return ip; // caller wants the disp32
1158 ip += 4; // skip the disp32
1159 }
1160 break;
1161
1162 case 1:
1163 // [01 reg 100][ss index base][disp8]
1164 // [01 reg 100][00 100 esp][disp8]
1165 // [01 reg base] [disp8]
1166 ip += 1; // skip the disp8
1167 break;
1168
1169 case 2:
1170 // [10 reg 100][ss index base][disp32]
1171 // [10 reg 100][00 100 esp][disp32]
1172 // [10 reg base] [disp32]
1173 if (which == disp32_operand)
1174 return ip; // caller wants the disp32
1175 ip += 4; // skip the disp32
1176 break;
1177
1178 case 3:
1179 // [11 reg base] (not a memory addressing mode)
1180 break;
1181 }
1182
1183 if (which == end_pc_operand) {
1184 return ip + tail_size;
1185 }
1186
1187 #ifdef _LP64
1188 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1189 #else
1190 assert(which == imm_operand, "instruction has only an imm field");
1191 #endif // LP64
1192 return ip;
1193 }
1194
1195 address Assembler::locate_next_instruction(address inst) {
1196 // Secretly share code with locate_operand:
1197 return locate_operand(inst, end_pc_operand);
1198 }
1199
1200
1201 #ifdef ASSERT
1202 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1203 address inst = inst_mark();
1204 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1205 address opnd;
1206
1207 Relocation* r = rspec.reloc();
1208 if (r->type() == relocInfo::none) {
1209 return;
1210 } else if (r->is_call() || format == call32_operand) {
1211 // assert(format == imm32_operand, "cannot specify a nonzero format");
1212 opnd = locate_operand(inst, call32_operand);
1213 } else if (r->is_data()) {
1214 assert(format == imm_operand || format == disp32_operand
1215 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1216 opnd = locate_operand(inst, (WhichOperand)format);
1217 } else {
1218 assert(format == imm_operand, "cannot specify a format");
1219 return;
1220 }
1221 assert(opnd == pc(), "must put operand where relocs can find it");
1222 }
1223 #endif // ASSERT
1224
1225 void Assembler::emit_operand(Register reg, Address adr,
1226 int rip_relative_correction) {
1227 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1228 adr._rspec,
1229 rip_relative_correction);
1230 }
1231
1232 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1233 if (adr.isxmmindex()) {
1234 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
1235 } else {
1236 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1237 adr._rspec);
1238 }
1239 }
1240
1241 // Now the Assembler instructions (identical for 32/64 bits)
1242
1243 void Assembler::adcl(Address dst, int32_t imm32) {
1244 InstructionMark im(this);
1245 prefix(dst);
1246 emit_arith_operand(0x81, rdx, dst, imm32);
1247 }
1248
1249 void Assembler::adcl(Address dst, Register src) {
1250 InstructionMark im(this);
1251 prefix(dst, src);
1252 emit_int8(0x11);
1253 emit_operand(src, dst);
1254 }
1255
1256 void Assembler::adcl(Register dst, int32_t imm32) {
1257 prefix(dst);
1258 emit_arith(0x81, 0xD0, dst, imm32);
1259 }
1260
1261 void Assembler::adcl(Register dst, Address src) {
1262 InstructionMark im(this);
1263 prefix(src, dst);
1264 emit_int8(0x13);
1265 emit_operand(dst, src);
1266 }
1267
1268 void Assembler::adcl(Register dst, Register src) {
1269 (void) prefix_and_encode(dst->encoding(), src->encoding());
1270 emit_arith(0x13, 0xC0, dst, src);
1271 }
1272
1273 void Assembler::addl(Address dst, int32_t imm32) {
1274 InstructionMark im(this);
1275 prefix(dst);
1276 emit_arith_operand(0x81, rax, dst, imm32);
1277 }
1278
1279 void Assembler::addb(Address dst, int imm8) {
1280 InstructionMark im(this);
1281 prefix(dst);
1282 emit_int8((unsigned char)0x80);
1283 emit_operand(rax, dst, 1);
1284 emit_int8(imm8);
1285 }
1286
1287 void Assembler::addw(Register dst, Register src) {
1288 (void)prefix_and_encode(dst->encoding(), src->encoding());
1289 emit_arith(0x03, 0xC0, dst, src);
1290 }
1291
1292 void Assembler::addw(Address dst, int imm16) {
1293 InstructionMark im(this);
1294 emit_int8(0x66);
1295 prefix(dst);
1296 emit_int8((unsigned char)0x81);
1297 emit_operand(rax, dst, 2);
1298 emit_int16(imm16);
1299 }
1300
1301 void Assembler::addl(Address dst, Register src) {
1302 InstructionMark im(this);
1303 prefix(dst, src);
1304 emit_int8(0x01);
1305 emit_operand(src, dst);
1306 }
1307
1308 void Assembler::addl(Register dst, int32_t imm32) {
1309 prefix(dst);
1310 emit_arith(0x81, 0xC0, dst, imm32);
1311 }
1312
1313 void Assembler::addl(Register dst, Address src) {
1314 InstructionMark im(this);
1315 prefix(src, dst);
1316 emit_int8(0x03);
1317 emit_operand(dst, src);
1318 }
1319
1320 void Assembler::addl(Register dst, Register src) {
1321 (void) prefix_and_encode(dst->encoding(), src->encoding());
1322 emit_arith(0x03, 0xC0, dst, src);
1323 }
1324
1325 void Assembler::addr_nop_4() {
1326 assert(UseAddressNop, "no CPU support");
1327 // 4 bytes: NOP DWORD PTR [EAX+0]
1328 emit_int32(0x0F,
1329 0x1F,
1330 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1331 0); // 8-bits offset (1 byte)
1332 }
1333
1334 void Assembler::addr_nop_5() {
1335 assert(UseAddressNop, "no CPU support");
1336 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1337 emit_int32(0x0F,
1338 0x1F,
1339 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1340 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1341 emit_int8(0); // 8-bits offset (1 byte)
1342 }
1343
1344 void Assembler::addr_nop_7() {
1345 assert(UseAddressNop, "no CPU support");
1346 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1347 emit_int24(0x0F,
1348 0x1F,
1349 (unsigned char)0x80);
1350 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1351 emit_int32(0); // 32-bits offset (4 bytes)
1352 }
1353
1354 void Assembler::addr_nop_8() {
1355 assert(UseAddressNop, "no CPU support");
1356 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1357 emit_int32(0x0F,
1358 0x1F,
1359 (unsigned char)0x84,
1360 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1361 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1362 emit_int32(0); // 32-bits offset (4 bytes)
1363 }
1364
1365 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1366 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1367 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1368 attributes.set_rex_vex_w_reverted();
1369 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1370 emit_int16(0x58, (0xC0 | encode));
1371 }
1372
1373 void Assembler::addsd(XMMRegister dst, Address src) {
1374 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1375 InstructionMark im(this);
1376 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1377 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1378 attributes.set_rex_vex_w_reverted();
1379 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1380 emit_int8(0x58);
1381 emit_operand(dst, src);
1382 }
1383
1384 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1385 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1386 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1387 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1388 emit_int16(0x58, (0xC0 | encode));
1389 }
1390
1391 void Assembler::addss(XMMRegister dst, Address src) {
1392 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1393 InstructionMark im(this);
1394 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1395 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1396 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1397 emit_int8(0x58);
1398 emit_operand(dst, src);
1399 }
1400
1401 void Assembler::aesdec(XMMRegister dst, Address src) {
1402 assert(VM_Version::supports_aes(), "");
1403 InstructionMark im(this);
1404 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1405 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1406 emit_int8((unsigned char)0xDE);
1407 emit_operand(dst, src);
1408 }
1409
1410 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1411 assert(VM_Version::supports_aes(), "");
1412 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1413 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1414 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1415 }
1416
1417 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1418 assert(VM_Version::supports_avx512_vaes(), "");
1419 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1420 attributes.set_is_evex_instruction();
1421 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1422 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1423 }
1424
1425
1426 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1427 assert(VM_Version::supports_aes(), "");
1428 InstructionMark im(this);
1429 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1430 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1431 emit_int8((unsigned char)0xDF);
1432 emit_operand(dst, src);
1433 }
1434
1435 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1436 assert(VM_Version::supports_aes(), "");
1437 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1438 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1439 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1440 }
1441
1442 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1443 assert(VM_Version::supports_avx512_vaes(), "");
1444 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1445 attributes.set_is_evex_instruction();
1446 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1447 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1448 }
1449
1450 void Assembler::aesenc(XMMRegister dst, Address src) {
1451 assert(VM_Version::supports_aes(), "");
1452 InstructionMark im(this);
1453 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1454 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1455 emit_int8((unsigned char)0xDC);
1456 emit_operand(dst, src);
1457 }
1458
1459 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1460 assert(VM_Version::supports_aes(), "");
1461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1462 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1463 emit_int16((unsigned char)0xDC, 0xC0 | encode);
1464 }
1465
1466 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1467 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1468 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1469 attributes.set_is_evex_instruction();
1470 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1471 emit_int16((unsigned char)0xDC, (0xC0 | encode));
1472 }
1473
1474 void Assembler::aesenclast(XMMRegister dst, Address src) {
1475 assert(VM_Version::supports_aes(), "");
1476 InstructionMark im(this);
1477 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1478 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1479 emit_int8((unsigned char)0xDD);
1480 emit_operand(dst, src);
1481 }
1482
1483 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1484 assert(VM_Version::supports_aes(), "");
1485 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1486 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1487 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1488 }
1489
1490 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1491 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1492 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1493 attributes.set_is_evex_instruction();
1494 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1495 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1496 }
1497
1498 void Assembler::andb(Address dst, Register src) {
1499 InstructionMark im(this);
1500 prefix(dst, src, true);
1501 emit_int8(0x20);
1502 emit_operand(src, dst);
1503 }
1504
1505 void Assembler::andw(Register dst, Register src) {
1506 (void)prefix_and_encode(dst->encoding(), src->encoding());
1507 emit_arith(0x23, 0xC0, dst, src);
1508 }
1509
1510 void Assembler::andl(Address dst, int32_t imm32) {
1511 InstructionMark im(this);
1512 prefix(dst);
1513 emit_arith_operand(0x81, as_Register(4), dst, imm32);
1514 }
1515
1516 void Assembler::andl(Register dst, int32_t imm32) {
1517 prefix(dst);
1518 emit_arith(0x81, 0xE0, dst, imm32);
1519 }
1520
1521 void Assembler::andl(Address dst, Register src) {
1522 InstructionMark im(this);
1523 prefix(dst, src);
1524 emit_int8(0x21);
1525 emit_operand(src, dst);
1526 }
1527
1528 void Assembler::andl(Register dst, Address src) {
1529 InstructionMark im(this);
1530 prefix(src, dst);
1531 emit_int8(0x23);
1532 emit_operand(dst, src);
1533 }
1534
1535 void Assembler::andl(Register dst, Register src) {
1536 (void) prefix_and_encode(dst->encoding(), src->encoding());
1537 emit_arith(0x23, 0xC0, dst, src);
1538 }
1539
1540 void Assembler::andnl(Register dst, Register src1, Register src2) {
1541 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1542 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1543 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1544 emit_int16((unsigned char)0xF2, (0xC0 | encode));
1545 }
1546
1547 void Assembler::andnl(Register dst, Register src1, Address src2) {
1548 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1549 InstructionMark im(this);
1550 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1551 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1552 emit_int8((unsigned char)0xF2);
1553 emit_operand(dst, src2);
1554 }
1555
1556 void Assembler::bsfl(Register dst, Register src) {
1557 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1558 emit_int24(0x0F,
1559 (unsigned char)0xBC,
1560 0xC0 | encode);
1561 }
1562
1563 void Assembler::bsrl(Register dst, Register src) {
1564 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1565 emit_int24(0x0F,
1566 (unsigned char)0xBD,
1567 0xC0 | encode);
1568 }
1569
1570 void Assembler::bswapl(Register reg) { // bswap
1571 int encode = prefix_and_encode(reg->encoding());
1572 emit_int16(0x0F, (0xC8 | encode));
1573 }
1574
1575 void Assembler::blsil(Register dst, Register src) {
1576 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1577 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1578 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1579 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1580 }
1581
1582 void Assembler::blsil(Register dst, Address src) {
1583 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1584 InstructionMark im(this);
1585 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1586 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1587 emit_int8((unsigned char)0xF3);
1588 emit_operand(rbx, src);
1589 }
1590
1591 void Assembler::blsmskl(Register dst, Register src) {
1592 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1593 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1594 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1595 emit_int16((unsigned char)0xF3,
1596 0xC0 | encode);
1597 }
1598
1599 void Assembler::blsmskl(Register dst, Address src) {
1600 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1601 InstructionMark im(this);
1602 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1603 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1604 emit_int8((unsigned char)0xF3);
1605 emit_operand(rdx, src);
1606 }
1607
1608 void Assembler::blsrl(Register dst, Register src) {
1609 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1610 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1611 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1612 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1613 }
1614
1615 void Assembler::blsrl(Register dst, Address src) {
1616 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1617 InstructionMark im(this);
1618 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1619 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1620 emit_int8((unsigned char)0xF3);
1621 emit_operand(rcx, src);
1622 }
1623
1624 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1625 // suspect disp32 is always good
1626 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1627
1628 if (L.is_bound()) {
1629 const int long_size = 5;
1630 int offs = (int)( target(L) - pc() );
1631 assert(offs <= 0, "assembler error");
1632 InstructionMark im(this);
1633 // 1110 1000 #32-bit disp
1634 emit_int8((unsigned char)0xE8);
1635 emit_data(offs - long_size, rtype, operand);
1636 } else {
1637 InstructionMark im(this);
1638 // 1110 1000 #32-bit disp
1639 L.add_patch_at(code(), locator());
1640
1641 emit_int8((unsigned char)0xE8);
1642 emit_data(int(0), rtype, operand);
1643 }
1644 }
1645
1646 void Assembler::call(Register dst) {
1647 int encode = prefix_and_encode(dst->encoding());
1648 emit_int16((unsigned char)0xFF, (0xD0 | encode));
1649 }
1650
1651
1652 void Assembler::call(Address adr) {
1653 InstructionMark im(this);
1654 prefix(adr);
1655 emit_int8((unsigned char)0xFF);
1656 emit_operand(rdx, adr);
1657 }
1658
1659 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1660 InstructionMark im(this);
1661 emit_int8((unsigned char)0xE8);
1662 intptr_t disp = entry - (pc() + sizeof(int32_t));
1663 // Entry is NULL in case of a scratch emit.
1664 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1665 // Technically, should use call32_operand, but this format is
1666 // implied by the fact that we're emitting a call instruction.
1667
1668 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1669 emit_data((int) disp, rspec, operand);
1670 }
1671
1672 void Assembler::cdql() {
1673 emit_int8((unsigned char)0x99);
1674 }
1675
1676 void Assembler::cld() {
1677 emit_int8((unsigned char)0xFC);
1678 }
1679
1680 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1681 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1682 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1683 emit_int24(0x0F,
1684 0x40 | cc,
1685 0xC0 | encode);
1686 }
1687
1688
1689 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1690 InstructionMark im(this);
1691 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1692 prefix(src, dst);
1693 emit_int16(0x0F, (0x40 | cc));
1694 emit_operand(dst, src);
1695 }
1696
1697 void Assembler::cmpb(Address dst, int imm8) {
1698 InstructionMark im(this);
1699 prefix(dst);
1700 emit_int8((unsigned char)0x80);
1701 emit_operand(rdi, dst, 1);
1702 emit_int8(imm8);
1703 }
1704
1705 void Assembler::cmpl(Address dst, int32_t imm32) {
1706 InstructionMark im(this);
1707 prefix(dst);
1708 emit_int8((unsigned char)0x81);
1709 emit_operand(rdi, dst, 4);
1710 emit_int32(imm32);
1711 }
1712
1713 void Assembler::cmpl(Register dst, int32_t imm32) {
1714 prefix(dst);
1715 emit_arith(0x81, 0xF8, dst, imm32);
1716 }
1717
1718 void Assembler::cmpl(Register dst, Register src) {
1719 (void) prefix_and_encode(dst->encoding(), src->encoding());
1720 emit_arith(0x3B, 0xC0, dst, src);
1721 }
1722
1723 void Assembler::cmpl(Register dst, Address src) {
1724 InstructionMark im(this);
1725 prefix(src, dst);
1726 emit_int8(0x3B);
1727 emit_operand(dst, src);
1728 }
1729
1730 void Assembler::cmpw(Address dst, int imm16) {
1731 InstructionMark im(this);
1732 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1733 emit_int16(0x66, (unsigned char)0x81);
1734 emit_operand(rdi, dst, 2);
1735 emit_int16(imm16);
1736 }
1737
1738 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1739 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1740 // The ZF is set if the compared values were equal, and cleared otherwise.
1741 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1742 InstructionMark im(this);
1743 prefix(adr, reg);
1744 emit_int16(0x0F, (unsigned char)0xB1);
1745 emit_operand(reg, adr);
1746 }
1747
1748 void Assembler::cmpxchgw(Register reg, Address adr) { // cmpxchg
1749 InstructionMark im(this);
1750 size_prefix();
1751 prefix(adr, reg);
1752 emit_int16(0x0F, (unsigned char)0xB1);
1753 emit_operand(reg, adr);
1754 }
1755
1756 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1757 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1758 // The ZF is set if the compared values were equal, and cleared otherwise.
1759 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1760 InstructionMark im(this);
1761 prefix(adr, reg, true);
1762 emit_int16(0x0F, (unsigned char)0xB0);
1763 emit_operand(reg, adr);
1764 }
1765
1766 void Assembler::comisd(XMMRegister dst, Address src) {
1767 // NOTE: dbx seems to decode this as comiss even though the
1768 // 0x66 is there. Strangely ucomisd comes out correct
1769 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1770 InstructionMark im(this);
1771 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1772 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1773 attributes.set_rex_vex_w_reverted();
1774 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1775 emit_int8(0x2F);
1776 emit_operand(dst, src);
1777 }
1778
1779 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1780 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1781 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1782 attributes.set_rex_vex_w_reverted();
1783 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1784 emit_int16(0x2F, (0xC0 | encode));
1785 }
1786
1787 void Assembler::comiss(XMMRegister dst, Address src) {
1788 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1789 InstructionMark im(this);
1790 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1791 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1792 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1793 emit_int8(0x2F);
1794 emit_operand(dst, src);
1795 }
1796
1797 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1798 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1799 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1800 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1801 emit_int16(0x2F, (0xC0 | encode));
1802 }
1803
1804 void Assembler::cpuid() {
1805 emit_int16(0x0F, (unsigned char)0xA2);
1806 }
1807
1808 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1809 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1810 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1811 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1812 //
1813 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1814 //
1815 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1816 //
1817 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1818 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1819 assert(VM_Version::supports_sse4_2(), "");
1820 int8_t w = 0x01;
1821 Prefix p = Prefix_EMPTY;
1822
1823 emit_int8((unsigned char)0xF2);
1824 switch (sizeInBytes) {
1825 case 1:
1826 w = 0;
1827 break;
1828 case 2:
1829 case 4:
1830 break;
1831 LP64_ONLY(case 8:)
1832 // This instruction is not valid in 32 bits
1833 // Note:
1834 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1835 //
1836 // Page B - 72 Vol. 2C says
1837 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1838 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1839 // F0!!!
1840 // while 3 - 208 Vol. 2A
1841 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1842 //
1843 // the 0 on a last bit is reserved for a different flavor of this instruction :
1844 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1845 p = REX_W;
1846 break;
1847 default:
1848 assert(0, "Unsupported value for a sizeInBytes argument");
1849 break;
1850 }
1851 LP64_ONLY(prefix(crc, v, p);)
1852 emit_int32(0x0F,
1853 0x38,
1854 0xF0 | w,
1855 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1856 }
1857
1858 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1859 assert(VM_Version::supports_sse4_2(), "");
1860 InstructionMark im(this);
1861 int8_t w = 0x01;
1862 Prefix p = Prefix_EMPTY;
1863
1864 emit_int8((int8_t)0xF2);
1865 switch (sizeInBytes) {
1866 case 1:
1867 w = 0;
1868 break;
1869 case 2:
1870 case 4:
1871 break;
1872 LP64_ONLY(case 8:)
1873 // This instruction is not valid in 32 bits
1874 p = REX_W;
1875 break;
1876 default:
1877 assert(0, "Unsupported value for a sizeInBytes argument");
1878 break;
1879 }
1880 LP64_ONLY(prefix(crc, adr, p);)
1881 emit_int24(0x0F, 0x38, (0xF0 | w));
1882 emit_operand(crc, adr);
1883 }
1884
1885 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1886 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1887 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1888 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1889 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1890 }
1891
1892 void Assembler::vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
1893 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1894 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1895 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1896 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1897 }
1898
1899 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1900 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1901 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1902 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1903 emit_int16(0x5B, (0xC0 | encode));
1904 }
1905
1906 void Assembler::vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
1907 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
1908 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1909 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1910 emit_int16(0x5B, (0xC0 | encode));
1911 }
1912
1913 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1914 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1915 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1916 attributes.set_rex_vex_w_reverted();
1917 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1918 emit_int16(0x5A, (0xC0 | encode));
1919 }
1920
1921 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1922 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1923 InstructionMark im(this);
1924 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1925 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1926 attributes.set_rex_vex_w_reverted();
1927 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1928 emit_int8(0x5A);
1929 emit_operand(dst, src);
1930 }
1931
1932 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1933 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1934 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1935 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1936 emit_int16(0x2A, (0xC0 | encode));
1937 }
1938
1939 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1940 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1941 InstructionMark im(this);
1942 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1943 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1944 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1945 emit_int8(0x2A);
1946 emit_operand(dst, src);
1947 }
1948
1949 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1950 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1951 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1952 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1953 emit_int16(0x2A, (0xC0 | encode));
1954 }
1955
1956 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1957 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1958 InstructionMark im(this);
1959 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1960 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1961 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1962 emit_int8(0x2A);
1963 emit_operand(dst, src);
1964 }
1965
1966 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1967 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1968 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1969 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1970 emit_int16(0x2A, (0xC0 | encode));
1971 }
1972
1973 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1974 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1975 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1976 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1977 emit_int16(0x5A, (0xC0 | encode));
1978 }
1979
1980 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1981 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1982 InstructionMark im(this);
1983 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1984 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1985 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1986 emit_int8(0x5A);
1987 emit_operand(dst, src);
1988 }
1989
1990
1991 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1992 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1993 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1994 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1995 emit_int16(0x2C, (0xC0 | encode));
1996 }
1997
1998 void Assembler::cvtss2sil(Register dst, XMMRegister src) {
1999 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2000 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2001 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2002 emit_int16(0x2D, (0xC0 | encode));
2003 }
2004
2005 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
2006 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2007 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2008 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2009 emit_int16(0x2C, (0xC0 | encode));
2010 }
2011
2012 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
2013 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2014 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2015 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2016 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2017 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2018 }
2019
2020 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
2021 assert(VM_Version::supports_ssse3(), "");
2022 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2023 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2024 emit_int16(0x1C, (0xC0 | encode));
2025 }
2026
2027 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
2028 assert(VM_Version::supports_ssse3(), "");
2029 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2030 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2031 emit_int16(0x1D, (0xC0 | encode));
2032 }
2033
2034 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
2035 assert(VM_Version::supports_ssse3(), "");
2036 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2037 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2038 emit_int16(0x1E, (0xC0 | encode));
2039 }
2040
2041 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
2042 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2043 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2044 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
2045 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2046 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2047 emit_int16(0x1C, (0xC0 | encode));
2048 }
2049
2050 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
2051 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
2052 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
2053 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "");
2054 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2055 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2056 emit_int16(0x1D, (0xC0 | encode));
2057 }
2058
2059 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
2060 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
2061 vector_len == AVX_256bit? VM_Version::supports_avx2() :
2062 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
2063 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2064 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2065 emit_int16(0x1E, (0xC0 | encode));
2066 }
2067
2068 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
2069 assert(UseAVX > 2, "");
2070 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2071 attributes.set_is_evex_instruction();
2072 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
2073 emit_int16(0x1F, (0xC0 | encode));
2074 }
2075
2076 void Assembler::vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2077 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2078 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2079 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2080 emit_int16(0x5A, (0xC0 | encode));
2081 }
2082
2083 void Assembler::vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2084 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2085 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2086 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2087 attributes.set_rex_vex_w_reverted();
2088 emit_int16(0x5A, (0xC0 | encode));
2089 }
2090
2091 void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2092 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2093 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2094 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2095 emit_int16(0x5B, (0xC0 | encode));
2096 }
2097
2098 void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
2099 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
2100 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2101 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2102 emit_int16(0x5B, (0xC0 | encode));
2103 }
2104
2105 void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2106 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
2107 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2108 attributes.set_is_evex_instruction();
2109 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2110 emit_int16(0x7A, (0xC0 | encode));
2111 }
2112
2113 void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2114 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
2115 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2116 attributes.set_is_evex_instruction();
2117 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2118 emit_int16(0x7B, (0xC0 | encode));
2119 }
2120
2121 void Assembler::evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len) {
2122 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
2123 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2124 attributes.set_is_evex_instruction();
2125 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2126 emit_int16(0x5B, (0xC0 | encode));
2127 }
2128
2129 void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
2130 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
2131 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2132 attributes.set_is_evex_instruction();
2133 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2134 emit_int16(0x7A, (0xC0 | encode));
2135 }
2136
2137 void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
2138 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "");
2139 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2140 attributes.set_is_evex_instruction();
2141 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2142 emit_int16((unsigned char)0xE6, (0xC0 | encode));
2143 }
2144
2145 void Assembler::evpmovwb(XMMRegister dst, XMMRegister src, int vector_len) {
2146 assert(UseAVX > 2 && VM_Version::supports_avx512bw(), "");
2147 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2148 attributes.set_is_evex_instruction();
2149 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2150 emit_int16(0x30, (0xC0 | encode));
2151 }
2152
2153 void Assembler::evpmovdw(XMMRegister dst, XMMRegister src, int vector_len) {
2154 assert(UseAVX > 2, "");
2155 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2156 attributes.set_is_evex_instruction();
2157 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2158 emit_int16(0x33, (0xC0 | encode));
2159 }
2160
2161 void Assembler::evpmovdb(XMMRegister dst, XMMRegister src, int vector_len) {
2162 assert(UseAVX > 2, "");
2163 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2164 attributes.set_is_evex_instruction();
2165 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2166 emit_int16(0x31, (0xC0 | encode));
2167 }
2168
2169 void Assembler::evpmovqd(XMMRegister dst, XMMRegister src, int vector_len) {
2170 assert(UseAVX > 2, "");
2171 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2172 attributes.set_is_evex_instruction();
2173 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2174 emit_int16(0x35, (0xC0 | encode));
2175 }
2176
2177 void Assembler::evpmovqb(XMMRegister dst, XMMRegister src, int vector_len) {
2178 assert(UseAVX > 2, "");
2179 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2180 attributes.set_is_evex_instruction();
2181 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2182 emit_int16(0x32, (0xC0 | encode));
2183 }
2184
2185 void Assembler::evpmovqw(XMMRegister dst, XMMRegister src, int vector_len) {
2186 assert(UseAVX > 2, "");
2187 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2188 attributes.set_is_evex_instruction();
2189 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2190 emit_int16(0x34, (0xC0 | encode));
2191 }
2192
2193 void Assembler::evpmovsqd(XMMRegister dst, XMMRegister src, int vector_len) {
2194 assert(UseAVX > 2, "");
2195 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2196 attributes.set_is_evex_instruction();
2197 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
2198 emit_int16(0x25, (0xC0 | encode));
2199 }
2200
2201 void Assembler::decl(Address dst) {
2202 // Don't use it directly. Use MacroAssembler::decrement() instead.
2203 InstructionMark im(this);
2204 prefix(dst);
2205 emit_int8((unsigned char)0xFF);
2206 emit_operand(rcx, dst);
2207 }
2208
2209 void Assembler::divsd(XMMRegister dst, Address src) {
2210 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2211 InstructionMark im(this);
2212 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2213 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2214 attributes.set_rex_vex_w_reverted();
2215 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2216 emit_int8(0x5E);
2217 emit_operand(dst, src);
2218 }
2219
2220 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
2221 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2222 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2223 attributes.set_rex_vex_w_reverted();
2224 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2225 emit_int16(0x5E, (0xC0 | encode));
2226 }
2227
2228 void Assembler::divss(XMMRegister dst, Address src) {
2229 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2230 InstructionMark im(this);
2231 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2232 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2233 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2234 emit_int8(0x5E);
2235 emit_operand(dst, src);
2236 }
2237
2238 void Assembler::divss(XMMRegister dst, XMMRegister src) {
2239 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2240 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2241 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2242 emit_int16(0x5E, (0xC0 | encode));
2243 }
2244
2245 void Assembler::hlt() {
2246 emit_int8((unsigned char)0xF4);
2247 }
2248
2249 void Assembler::idivl(Register src) {
2250 int encode = prefix_and_encode(src->encoding());
2251 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2252 }
2253
2254 void Assembler::divl(Register src) { // Unsigned
2255 int encode = prefix_and_encode(src->encoding());
2256 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2257 }
2258
2259 void Assembler::imull(Register src) {
2260 int encode = prefix_and_encode(src->encoding());
2261 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2262 }
2263
2264 void Assembler::imull(Register dst, Register src) {
2265 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2266 emit_int24(0x0F,
2267 (unsigned char)0xAF,
2268 (0xC0 | encode));
2269 }
2270
2271 void Assembler::imull(Register dst, Address src, int32_t value) {
2272 InstructionMark im(this);
2273 prefix(src, dst);
2274 if (is8bit(value)) {
2275 emit_int8((unsigned char)0x6B);
2276 emit_operand(dst, src);
2277 emit_int8(value);
2278 } else {
2279 emit_int8((unsigned char)0x69);
2280 emit_operand(dst, src);
2281 emit_int32(value);
2282 }
2283 }
2284
2285 void Assembler::imull(Register dst, Register src, int value) {
2286 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2287 if (is8bit(value)) {
2288 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2289 } else {
2290 emit_int16(0x69, (0xC0 | encode));
2291 emit_int32(value);
2292 }
2293 }
2294
2295 void Assembler::imull(Register dst, Address src) {
2296 InstructionMark im(this);
2297 prefix(src, dst);
2298 emit_int16(0x0F, (unsigned char)0xAF);
2299 emit_operand(dst, src);
2300 }
2301
2302
2303 void Assembler::incl(Address dst) {
2304 // Don't use it directly. Use MacroAssembler::increment() instead.
2305 InstructionMark im(this);
2306 prefix(dst);
2307 emit_int8((unsigned char)0xFF);
2308 emit_operand(rax, dst);
2309 }
2310
2311 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2312 InstructionMark im(this);
2313 assert((0 <= cc) && (cc < 16), "illegal cc");
2314 if (L.is_bound()) {
2315 address dst = target(L);
2316 assert(dst != NULL, "jcc most probably wrong");
2317
2318 const int short_size = 2;
2319 const int long_size = 6;
2320 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2321 if (maybe_short && is8bit(offs - short_size)) {
2322 // 0111 tttn #8-bit disp
2323 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2324 } else {
2325 // 0000 1111 1000 tttn #32-bit disp
2326 assert(is_simm32(offs - long_size),
2327 "must be 32bit offset (call4)");
2328 emit_int16(0x0F, (0x80 | cc));
2329 emit_int32(offs - long_size);
2330 }
2331 } else {
2332 // Note: could eliminate cond. jumps to this jump if condition
2333 // is the same however, seems to be rather unlikely case.
2334 // Note: use jccb() if label to be bound is very close to get
2335 // an 8-bit displacement
2336 L.add_patch_at(code(), locator());
2337 emit_int16(0x0F, (0x80 | cc));
2338 emit_int32(0);
2339 }
2340 }
2341
2342 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2343 if (L.is_bound()) {
2344 const int short_size = 2;
2345 address entry = target(L);
2346 #ifdef ASSERT
2347 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2348 intptr_t delta = short_branch_delta();
2349 if (delta != 0) {
2350 dist += (dist < 0 ? (-delta) :delta);
2351 }
2352 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2353 #endif
2354 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2355 // 0111 tttn #8-bit disp
2356 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2357 } else {
2358 InstructionMark im(this);
2359 L.add_patch_at(code(), locator(), file, line);
2360 emit_int16(0x70 | cc, 0);
2361 }
2362 }
2363
2364 void Assembler::jmp(Address adr) {
2365 InstructionMark im(this);
2366 prefix(adr);
2367 emit_int8((unsigned char)0xFF);
2368 emit_operand(rsp, adr);
2369 }
2370
2371 void Assembler::jmp(Label& L, bool maybe_short) {
2372 if (L.is_bound()) {
2373 address entry = target(L);
2374 assert(entry != NULL, "jmp most probably wrong");
2375 InstructionMark im(this);
2376 const int short_size = 2;
2377 const int long_size = 5;
2378 intptr_t offs = entry - pc();
2379 if (maybe_short && is8bit(offs - short_size)) {
2380 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
2381 } else {
2382 emit_int8((unsigned char)0xE9);
2383 emit_int32(offs - long_size);
2384 }
2385 } else {
2386 // By default, forward jumps are always 32-bit displacements, since
2387 // we can't yet know where the label will be bound. If you're sure that
2388 // the forward jump will not run beyond 256 bytes, use jmpb to
2389 // force an 8-bit displacement.
2390 InstructionMark im(this);
2391 L.add_patch_at(code(), locator());
2392 emit_int8((unsigned char)0xE9);
2393 emit_int32(0);
2394 }
2395 }
2396
2397 void Assembler::jmp(Register entry) {
2398 int encode = prefix_and_encode(entry->encoding());
2399 emit_int16((unsigned char)0xFF, (0xE0 | encode));
2400 }
2401
2402 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2403 InstructionMark im(this);
2404 emit_int8((unsigned char)0xE9);
2405 assert(dest != NULL, "must have a target");
2406 intptr_t disp = dest - (pc() + sizeof(int32_t));
2407 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2408 emit_data(disp, rspec.reloc(), call32_operand);
2409 }
2410
2411 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2412 if (L.is_bound()) {
2413 const int short_size = 2;
2414 address entry = target(L);
2415 assert(entry != NULL, "jmp most probably wrong");
2416 #ifdef ASSERT
2417 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2418 intptr_t delta = short_branch_delta();
2419 if (delta != 0) {
2420 dist += (dist < 0 ? (-delta) :delta);
2421 }
2422 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2423 #endif
2424 intptr_t offs = entry - pc();
2425 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
2426 } else {
2427 InstructionMark im(this);
2428 L.add_patch_at(code(), locator(), file, line);
2429 emit_int16((unsigned char)0xEB, 0);
2430 }
2431 }
2432
2433 void Assembler::ldmxcsr( Address src) {
2434 if (UseAVX > 0 ) {
2435 InstructionMark im(this);
2436 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2437 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2438 emit_int8((unsigned char)0xAE);
2439 emit_operand(as_Register(2), src);
2440 } else {
2441 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2442 InstructionMark im(this);
2443 prefix(src);
2444 emit_int16(0x0F, (unsigned char)0xAE);
2445 emit_operand(as_Register(2), src);
2446 }
2447 }
2448
2449 void Assembler::leal(Register dst, Address src) {
2450 InstructionMark im(this);
2451 prefix(src, dst);
2452 emit_int8((unsigned char)0x8D);
2453 emit_operand(dst, src);
2454 }
2455
2456 void Assembler::lfence() {
2457 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
2458 }
2459
2460 void Assembler::lock() {
2461 emit_int8((unsigned char)0xF0);
2462 }
2463
2464 void Assembler::size_prefix() {
2465 emit_int8(0x66);
2466 }
2467
2468 void Assembler::lzcntl(Register dst, Register src) {
2469 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2470 emit_int8((unsigned char)0xF3);
2471 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2472 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
2473 }
2474
2475 void Assembler::lzcntl(Register dst, Address src) {
2476 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2477 InstructionMark im(this);
2478 emit_int8((unsigned char)0xF3);
2479 prefix(src, dst);
2480 emit_int16(0x0F, (unsigned char)0xBD);
2481 emit_operand(dst, src);
2482 }
2483
2484 // Emit mfence instruction
2485 void Assembler::mfence() {
2486 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2487 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
2488 }
2489
2490 // Emit sfence instruction
2491 void Assembler::sfence() {
2492 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2493 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
2494 }
2495
2496 void Assembler::mov(Register dst, Register src) {
2497 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2498 }
2499
2500 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2501 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2502 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2503 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2504 attributes.set_rex_vex_w_reverted();
2505 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2506 emit_int16(0x28, (0xC0 | encode));
2507 }
2508
2509 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2510 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2511 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2512 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2513 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2514 emit_int16(0x28, (0xC0 | encode));
2515 }
2516
2517 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2518 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2519 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2520 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2521 emit_int16(0x16, (0xC0 | encode));
2522 }
2523
2524 void Assembler::movb(Register dst, Address src) {
2525 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2526 InstructionMark im(this);
2527 prefix(src, dst, true);
2528 emit_int8((unsigned char)0x8A);
2529 emit_operand(dst, src);
2530 }
2531
2532 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2533 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2534 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2535 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2536 attributes.set_rex_vex_w_reverted();
2537 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2538 emit_int16(0x12, 0xC0 | encode);
2539 }
2540
2541 void Assembler::vmovddup(XMMRegister dst, Address src, int vector_len) {
2542 assert(VM_Version::supports_avx(), "");
2543 InstructionMark im(this);
2544 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2545 attributes.set_rex_vex_w_reverted();
2546 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2547 emit_int8(0x12);
2548 emit_operand(dst, src);
2549 }
2550
2551 void Assembler::kmovbl(KRegister dst, KRegister src) {
2552 assert(VM_Version::supports_avx512dq(), "");
2553 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2554 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2555 emit_int16((unsigned char)0x90, (0xC0 | encode));
2556 }
2557
2558 void Assembler::kmovbl(KRegister dst, Register src) {
2559 assert(VM_Version::supports_avx512dq(), "");
2560 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2561 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2562 emit_int16((unsigned char)0x92, (0xC0 | encode));
2563 }
2564
2565 void Assembler::kmovbl(Register dst, KRegister src) {
2566 assert(VM_Version::supports_avx512dq(), "");
2567 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2568 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2569 emit_int16((unsigned char)0x93, (0xC0 | encode));
2570 }
2571
2572 void Assembler::kmovwl(KRegister dst, Register src) {
2573 assert(VM_Version::supports_evex(), "");
2574 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2575 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2576 emit_int16((unsigned char)0x92, (0xC0 | encode));
2577 }
2578
2579 void Assembler::kmovwl(Register dst, KRegister src) {
2580 assert(VM_Version::supports_evex(), "");
2581 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2582 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2583 emit_int16((unsigned char)0x93, (0xC0 | encode));
2584 }
2585
2586 void Assembler::kmovwl(KRegister dst, Address src) {
2587 assert(VM_Version::supports_evex(), "");
2588 InstructionMark im(this);
2589 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2590 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2591 emit_int8((unsigned char)0x90);
2592 emit_operand((Register)dst, src);
2593 }
2594
2595 void Assembler::kmovwl(Address dst, KRegister src) {
2596 assert(VM_Version::supports_evex(), "");
2597 InstructionMark im(this);
2598 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2599 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2600 emit_int8((unsigned char)0x91);
2601 emit_operand((Register)src, dst);
2602 }
2603
2604 void Assembler::kmovwl(KRegister dst, KRegister src) {
2605 assert(VM_Version::supports_evex(), "");
2606 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2607 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2608 emit_int16((unsigned char)0x90, (0xC0 | encode));
2609 }
2610
2611 void Assembler::kmovdl(KRegister dst, Register src) {
2612 assert(VM_Version::supports_avx512bw(), "");
2613 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2614 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2615 emit_int16((unsigned char)0x92, (0xC0 | encode));
2616 }
2617
2618 void Assembler::kmovdl(Register dst, KRegister src) {
2619 assert(VM_Version::supports_avx512bw(), "");
2620 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2621 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2622 emit_int16((unsigned char)0x93, (0xC0 | encode));
2623 }
2624
2625 void Assembler::kmovql(KRegister dst, KRegister src) {
2626 assert(VM_Version::supports_avx512bw(), "");
2627 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2628 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2629 emit_int16((unsigned char)0x90, (0xC0 | encode));
2630 }
2631
2632 void Assembler::kmovql(KRegister dst, Address src) {
2633 assert(VM_Version::supports_avx512bw(), "");
2634 InstructionMark im(this);
2635 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2636 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2637 emit_int8((unsigned char)0x90);
2638 emit_operand((Register)dst, src);
2639 }
2640
2641 void Assembler::kmovql(Address dst, KRegister src) {
2642 assert(VM_Version::supports_avx512bw(), "");
2643 InstructionMark im(this);
2644 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2645 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2646 emit_int8((unsigned char)0x91);
2647 emit_operand((Register)src, dst);
2648 }
2649
2650 void Assembler::kmovql(KRegister dst, Register src) {
2651 assert(VM_Version::supports_avx512bw(), "");
2652 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2653 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2654 emit_int16((unsigned char)0x92, (0xC0 | encode));
2655 }
2656
2657 void Assembler::kmovql(Register dst, KRegister src) {
2658 assert(VM_Version::supports_avx512bw(), "");
2659 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2660 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2661 emit_int16((unsigned char)0x93, (0xC0 | encode));
2662 }
2663
2664 void Assembler::knotwl(KRegister dst, KRegister src) {
2665 assert(VM_Version::supports_evex(), "");
2666 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2667 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2668 emit_int16(0x44, (0xC0 | encode));
2669 }
2670
2671 void Assembler::knotbl(KRegister dst, KRegister src) {
2672 assert(VM_Version::supports_evex(), "");
2673 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2674 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2675 emit_int16(0x44, (0xC0 | encode));
2676 }
2677
2678 void Assembler::korbl(KRegister dst, KRegister src1, KRegister src2) {
2679 assert(VM_Version::supports_avx512dq(), "");
2680 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2681 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2682 emit_int16(0x45, (0xC0 | encode));
2683 }
2684
2685 void Assembler::korwl(KRegister dst, KRegister src1, KRegister src2) {
2686 assert(VM_Version::supports_evex(), "");
2687 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2688 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2689 emit_int16(0x45, (0xC0 | encode));
2690 }
2691
2692 void Assembler::kordl(KRegister dst, KRegister src1, KRegister src2) {
2693 assert(VM_Version::supports_avx512bw(), "");
2694 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2695 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2696 emit_int16(0x45, (0xC0 | encode));
2697 }
2698
2699 void Assembler::korql(KRegister dst, KRegister src1, KRegister src2) {
2700 assert(VM_Version::supports_avx512bw(), "");
2701 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2702 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2703 emit_int16(0x45, (0xC0 | encode));
2704 }
2705
2706 void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
2707 assert(VM_Version::supports_avx512dq(), "");
2708 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2709 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2710 emit_int16(0x47, (0xC0 | encode));
2711 }
2712
2713 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
2714 assert(VM_Version::supports_evex(), "");
2715 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2716 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2717 emit_int16(0x47, (0xC0 | encode));
2718 }
2719
2720 void Assembler::kxordl(KRegister dst, KRegister src1, KRegister src2) {
2721 assert(VM_Version::supports_avx512bw(), "");
2722 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2723 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2724 emit_int16(0x47, (0xC0 | encode));
2725 }
2726
2727 void Assembler::kxorql(KRegister dst, KRegister src1, KRegister src2) {
2728 assert(VM_Version::supports_avx512bw(), "");
2729 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2730 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2731 emit_int16(0x47, (0xC0 | encode));
2732 }
2733
2734 void Assembler::kandbl(KRegister dst, KRegister src1, KRegister src2) {
2735 assert(VM_Version::supports_avx512dq(), "");
2736 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2737 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2738 emit_int16(0x41, (0xC0 | encode));
2739 }
2740
2741 void Assembler::kandwl(KRegister dst, KRegister src1, KRegister src2) {
2742 assert(VM_Version::supports_evex(), "");
2743 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2744 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2745 emit_int16(0x41, (0xC0 | encode));
2746 }
2747
2748 void Assembler::kanddl(KRegister dst, KRegister src1, KRegister src2) {
2749 assert(VM_Version::supports_avx512bw(), "");
2750 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2751 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2752 emit_int16(0x41, (0xC0 | encode));
2753 }
2754
2755 void Assembler::kandql(KRegister dst, KRegister src1, KRegister src2) {
2756 assert(VM_Version::supports_avx512bw(), "");
2757 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2758 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2759 emit_int16(0x41, (0xC0 | encode));
2760 }
2761
2762 void Assembler::knotdl(KRegister dst, KRegister src) {
2763 assert(VM_Version::supports_avx512bw(), "");
2764 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2765 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2766 emit_int16(0x44, (0xC0 | encode));
2767 }
2768
2769 void Assembler::knotql(KRegister dst, KRegister src) {
2770 assert(VM_Version::supports_avx512bw(), "");
2771 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2772 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2773 emit_int16(0x44, (0xC0 | encode));
2774 }
2775
2776 // This instruction produces ZF or CF flags
2777 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2778 assert(VM_Version::supports_avx512dq(), "");
2779 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2780 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2781 emit_int16((unsigned char)0x98, (0xC0 | encode));
2782 }
2783
2784 // This instruction produces ZF or CF flags
2785 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2786 assert(VM_Version::supports_evex(), "");
2787 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2788 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2789 emit_int16((unsigned char)0x98, (0xC0 | encode));
2790 }
2791
2792 // This instruction produces ZF or CF flags
2793 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2794 assert(VM_Version::supports_avx512bw(), "");
2795 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2796 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2797 emit_int16((unsigned char)0x98, (0xC0 | encode));
2798 }
2799
2800 // This instruction produces ZF or CF flags
2801 void Assembler::kortestql(KRegister src1, KRegister src2) {
2802 assert(VM_Version::supports_avx512bw(), "");
2803 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2804 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2805 emit_int16((unsigned char)0x98, (0xC0 | encode));
2806 }
2807
2808 // This instruction produces ZF or CF flags
2809 void Assembler::ktestql(KRegister src1, KRegister src2) {
2810 assert(VM_Version::supports_avx512bw(), "");
2811 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2812 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2813 emit_int16((unsigned char)0x99, (0xC0 | encode));
2814 }
2815
2816 void Assembler::ktestdl(KRegister src1, KRegister src2) {
2817 assert(VM_Version::supports_avx512bw(), "");
2818 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2819 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2820 emit_int16((unsigned char)0x99, (0xC0 | encode));
2821 }
2822
2823 void Assembler::ktestwl(KRegister src1, KRegister src2) {
2824 assert(VM_Version::supports_avx512dq(), "");
2825 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2826 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2827 emit_int16((unsigned char)0x99, (0xC0 | encode));
2828 }
2829
2830 void Assembler::ktestbl(KRegister src1, KRegister src2) {
2831 assert(VM_Version::supports_avx512dq(), "");
2832 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2833 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2834 emit_int16((unsigned char)0x99, (0xC0 | encode));
2835 }
2836
2837 void Assembler::ktestq(KRegister src1, KRegister src2) {
2838 assert(VM_Version::supports_avx512bw(), "");
2839 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2840 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2841 emit_int16((unsigned char)0x99, (0xC0 | encode));
2842 }
2843
2844 void Assembler::ktestd(KRegister src1, KRegister src2) {
2845 assert(VM_Version::supports_avx512bw(), "");
2846 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2847 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2848 emit_int16((unsigned char)0x99, (0xC0 | encode));
2849 }
2850
2851 void Assembler::kxnorbl(KRegister dst, KRegister src1, KRegister src2) {
2852 assert(VM_Version::supports_avx512dq(), "");
2853 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2854 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2855 emit_int16(0x46, (0xC0 | encode));
2856 }
2857
2858 void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) {
2859 assert(VM_Version::supports_avx512dq(), "");
2860 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2861 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2862 emit_int16(0x32, (0xC0 | encode));
2863 emit_int8(imm8);
2864 }
2865
2866 void Assembler::kshiftlql(KRegister dst, KRegister src, int imm8) {
2867 assert(VM_Version::supports_avx512bw(), "");
2868 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2869 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2870 emit_int16(0x33, (0xC0 | encode));
2871 emit_int8(imm8);
2872 }
2873
2874
2875 void Assembler::kshiftrbl(KRegister dst, KRegister src, int imm8) {
2876 assert(VM_Version::supports_avx512dq(), "");
2877 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2878 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2879 emit_int16(0x30, (0xC0 | encode));
2880 }
2881
2882 void Assembler::kshiftrwl(KRegister dst, KRegister src, int imm8) {
2883 assert(VM_Version::supports_evex(), "");
2884 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2885 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2886 emit_int16(0x30, (0xC0 | encode));
2887 emit_int8(imm8);
2888 }
2889
2890 void Assembler::kshiftrdl(KRegister dst, KRegister src, int imm8) {
2891 assert(VM_Version::supports_avx512bw(), "");
2892 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2893 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2894 emit_int16(0x31, (0xC0 | encode));
2895 emit_int8(imm8);
2896 }
2897
2898 void Assembler::kshiftrql(KRegister dst, KRegister src, int imm8) {
2899 assert(VM_Version::supports_avx512bw(), "");
2900 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2901 int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
2902 emit_int16(0x31, (0xC0 | encode));
2903 emit_int8(imm8);
2904 }
2905
2906 void Assembler::kunpckdql(KRegister dst, KRegister src1, KRegister src2) {
2907 assert(VM_Version::supports_avx512bw(), "");
2908 InstructionAttr attributes(AVX_256bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2909 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2910 emit_int16(0x4B, (0xC0 | encode));
2911 }
2912
2913 void Assembler::movb(Address dst, int imm8) {
2914 InstructionMark im(this);
2915 prefix(dst);
2916 emit_int8((unsigned char)0xC6);
2917 emit_operand(rax, dst, 1);
2918 emit_int8(imm8);
2919 }
2920
2921
2922 void Assembler::movb(Address dst, Register src) {
2923 assert(src->has_byte_register(), "must have byte register");
2924 InstructionMark im(this);
2925 prefix(dst, src, true);
2926 emit_int8((unsigned char)0x88);
2927 emit_operand(src, dst);
2928 }
2929
2930 void Assembler::movdl(XMMRegister dst, Register src) {
2931 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2932 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2933 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2934 emit_int16(0x6E, (0xC0 | encode));
2935 }
2936
2937 void Assembler::movdl(Register dst, XMMRegister src) {
2938 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2939 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2940 // swap src/dst to get correct prefix
2941 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2942 emit_int16(0x7E, (0xC0 | encode));
2943 }
2944
2945 void Assembler::movdl(XMMRegister dst, Address src) {
2946 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2947 InstructionMark im(this);
2948 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2949 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2950 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2951 emit_int8(0x6E);
2952 emit_operand(dst, src);
2953 }
2954
2955 void Assembler::movdl(Address dst, XMMRegister src) {
2956 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2957 InstructionMark im(this);
2958 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2959 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2960 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2961 emit_int8(0x7E);
2962 emit_operand(src, dst);
2963 }
2964
2965 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2966 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2967 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2968 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2969 emit_int16(0x6F, (0xC0 | encode));
2970 }
2971
2972 void Assembler::movdqa(XMMRegister dst, Address src) {
2973 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2974 InstructionMark im(this);
2975 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2976 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2977 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2978 emit_int8(0x6F);
2979 emit_operand(dst, src);
2980 }
2981
2982 void Assembler::movdqu(XMMRegister dst, Address src) {
2983 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2984 InstructionMark im(this);
2985 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2986 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2987 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2988 emit_int8(0x6F);
2989 emit_operand(dst, src);
2990 }
2991
2992 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2993 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2994 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2995 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2996 emit_int16(0x6F, (0xC0 | encode));
2997 }
2998
2999 void Assembler::movdqu(Address dst, XMMRegister src) {
3000 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3001 InstructionMark im(this);
3002 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3003 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3004 attributes.reset_is_clear_context();
3005 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3006 emit_int8(0x7F);
3007 emit_operand(src, dst);
3008 }
3009
3010 // Move Unaligned 256bit Vector
3011 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3012 assert(UseAVX > 0, "");
3013 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3014 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3015 emit_int16(0x6F, (0xC0 | encode));
3016 }
3017
3018 void Assembler::vmovdqu(XMMRegister dst, Address src) {
3019 assert(UseAVX > 0, "");
3020 InstructionMark im(this);
3021 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3022 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3023 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3024 emit_int8(0x6F);
3025 emit_operand(dst, src);
3026 }
3027
3028 void Assembler::vmovdqu(Address dst, XMMRegister src) {
3029 assert(UseAVX > 0, "");
3030 InstructionMark im(this);
3031 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3032 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3033 attributes.reset_is_clear_context();
3034 // swap src<->dst for encoding
3035 assert(src != xnoreg, "sanity");
3036 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3037 emit_int8(0x7F);
3038 emit_operand(src, dst);
3039 }
3040
3041 void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3042 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3043 InstructionMark im(this);
3044 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3045 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3046 emit_int8((unsigned char)0x8C);
3047 emit_operand(dst, src);
3048 }
3049
3050 void Assembler::vpmaskmovq(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
3051 assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
3052 InstructionMark im(this);
3053 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3054 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3055 emit_int8((unsigned char)0x8C);
3056 emit_operand(dst, src);
3057 }
3058
3059 void Assembler::vmaskmovps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3060 assert(UseAVX > 0, "requires some form of AVX");
3061 InstructionMark im(this);
3062 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3063 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3064 emit_int8(0x2C);
3065 emit_operand(dst, src);
3066 }
3067
3068 void Assembler::vmaskmovpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
3069 assert(UseAVX > 0, "requires some form of AVX");
3070 InstructionMark im(this);
3071 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3072 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3073 emit_int8(0x2D);
3074 emit_operand(dst, src);
3075 }
3076
3077 void Assembler::vmaskmovps(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3078 assert(UseAVX > 0, "");
3079 InstructionMark im(this);
3080 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3081 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3082 emit_int8(0x2E);
3083 emit_operand(src, dst);
3084 }
3085
3086 void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int vector_len) {
3087 assert(UseAVX > 0, "");
3088 InstructionMark im(this);
3089 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3090 vex_prefix(dst, mask->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3091 emit_int8(0x2F);
3092 emit_operand(src, dst);
3093 }
3094
3095 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
3096 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3097 assert(VM_Version::supports_avx512vlbw(), "");
3098 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3099 attributes.set_embedded_opmask_register_specifier(mask);
3100 attributes.set_is_evex_instruction();
3101 if (merge) {
3102 attributes.reset_is_clear_context();
3103 }
3104 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3105 emit_int16(0x6F, (0xC0 | encode));
3106 }
3107
3108 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
3109 // Unmasked instruction
3110 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3111 }
3112
3113 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3114 assert(VM_Version::supports_avx512vlbw(), "");
3115 InstructionMark im(this);
3116 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3117 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3118 attributes.set_embedded_opmask_register_specifier(mask);
3119 attributes.set_is_evex_instruction();
3120 if (merge) {
3121 attributes.reset_is_clear_context();
3122 }
3123 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3124 emit_int8(0x6F);
3125 emit_operand(dst, src);
3126 }
3127
3128 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
3129 // Unmasked instruction
3130 evmovdqub(dst, k0, src, /*merge*/ false, vector_len);
3131 }
3132
3133 void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3134 assert(VM_Version::supports_avx512vlbw(), "");
3135 assert(src != xnoreg, "sanity");
3136 InstructionMark im(this);
3137 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3138 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3139 attributes.set_embedded_opmask_register_specifier(mask);
3140 attributes.set_is_evex_instruction();
3141 if (merge) {
3142 attributes.reset_is_clear_context();
3143 }
3144 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3145 emit_int8(0x7F);
3146 emit_operand(src, dst);
3147 }
3148
3149 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
3150 // Unmasked instruction
3151 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3152 }
3153
3154 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3155 assert(VM_Version::supports_avx512vlbw(), "");
3156 InstructionMark im(this);
3157 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3158 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3159 attributes.set_embedded_opmask_register_specifier(mask);
3160 attributes.set_is_evex_instruction();
3161 if (merge) {
3162 attributes.reset_is_clear_context();
3163 }
3164 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3165 emit_int8(0x6F);
3166 emit_operand(dst, src);
3167 }
3168
3169 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
3170 // Unmasked instruction
3171 evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
3172 }
3173
3174 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3175 assert(VM_Version::supports_avx512vlbw(), "");
3176 assert(src != xnoreg, "sanity");
3177 InstructionMark im(this);
3178 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
3179 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3180 attributes.set_embedded_opmask_register_specifier(mask);
3181 attributes.set_is_evex_instruction();
3182 if (merge) {
3183 attributes.reset_is_clear_context();
3184 }
3185 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3186 emit_int8(0x7F);
3187 emit_operand(src, dst);
3188 }
3189
3190 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
3191 // Unmasked instruction
3192 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3193 }
3194
3195 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3196 assert(VM_Version::supports_evex(), "");
3197 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3198 attributes.set_embedded_opmask_register_specifier(mask);
3199 attributes.set_is_evex_instruction();
3200 if (merge) {
3201 attributes.reset_is_clear_context();
3202 }
3203 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3204 emit_int16(0x6F, (0xC0 | encode));
3205 }
3206
3207 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
3208 // Unmasked instruction
3209 evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
3210 }
3211
3212 void Assembler::evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3213 assert(VM_Version::supports_evex(), "");
3214 InstructionMark im(this);
3215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
3216 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3217 attributes.set_embedded_opmask_register_specifier(mask);
3218 attributes.set_is_evex_instruction();
3219 if (merge) {
3220 attributes.reset_is_clear_context();
3221 }
3222 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3223 emit_int8(0x6F);
3224 emit_operand(dst, src);
3225 }
3226
3227 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
3228 // Unmasked isntruction
3229 evmovdqul(dst, k0, src, /*merge*/ true, vector_len);
3230 }
3231
3232 void Assembler::evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3233 assert(VM_Version::supports_evex(), "");
3234 assert(src != xnoreg, "sanity");
3235 InstructionMark im(this);
3236 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3237 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3238 attributes.set_embedded_opmask_register_specifier(mask);
3239 attributes.set_is_evex_instruction();
3240 if (merge) {
3241 attributes.reset_is_clear_context();
3242 }
3243 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3244 emit_int8(0x7F);
3245 emit_operand(src, dst);
3246 }
3247
3248 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
3249 // Unmasked instruction
3250 if (dst->encoding() == src->encoding()) return;
3251 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3252 }
3253
3254 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3255 assert(VM_Version::supports_evex(), "");
3256 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3257 attributes.set_embedded_opmask_register_specifier(mask);
3258 attributes.set_is_evex_instruction();
3259 if (merge) {
3260 attributes.reset_is_clear_context();
3261 }
3262 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3263 emit_int16(0x6F, (0xC0 | encode));
3264 }
3265
3266 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
3267 // Unmasked instruction
3268 evmovdquq(dst, k0, src, /*merge*/ false, vector_len);
3269 }
3270
3271 void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
3272 assert(VM_Version::supports_evex(), "");
3273 InstructionMark im(this);
3274 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3275 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3276 attributes.set_embedded_opmask_register_specifier(mask);
3277 attributes.set_is_evex_instruction();
3278 if (merge) {
3279 attributes.reset_is_clear_context();
3280 }
3281 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3282 emit_int8(0x6F);
3283 emit_operand(dst, src);
3284 }
3285
3286 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
3287 // Unmasked instruction
3288 evmovdquq(dst, k0, src, /*merge*/ true, vector_len);
3289 }
3290
3291 void Assembler::evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
3292 assert(VM_Version::supports_evex(), "");
3293 assert(src != xnoreg, "sanity");
3294 InstructionMark im(this);
3295 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3296 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3297 attributes.set_embedded_opmask_register_specifier(mask);
3298 if (merge) {
3299 attributes.reset_is_clear_context();
3300 }
3301 attributes.set_is_evex_instruction();
3302 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3303 emit_int8(0x7F);
3304 emit_operand(src, dst);
3305 }
3306
3307 // Uses zero extension on 64bit
3308
3309 void Assembler::movl(Register dst, int32_t imm32) {
3310 int encode = prefix_and_encode(dst->encoding());
3311 emit_int8(0xB8 | encode);
3312 emit_int32(imm32);
3313 }
3314
3315 void Assembler::movl(Register dst, Register src) {
3316 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3317 emit_int16((unsigned char)0x8B, (0xC0 | encode));
3318 }
3319
3320 void Assembler::movl(Register dst, Address src) {
3321 InstructionMark im(this);
3322 prefix(src, dst);
3323 emit_int8((unsigned char)0x8B);
3324 emit_operand(dst, src);
3325 }
3326
3327 void Assembler::movl(Address dst, int32_t imm32) {
3328 InstructionMark im(this);
3329 prefix(dst);
3330 emit_int8((unsigned char)0xC7);
3331 emit_operand(rax, dst, 4);
3332 emit_int32(imm32);
3333 }
3334
3335 void Assembler::movl(Address dst, Register src) {
3336 InstructionMark im(this);
3337 prefix(dst, src);
3338 emit_int8((unsigned char)0x89);
3339 emit_operand(src, dst);
3340 }
3341
3342 // New cpus require to use movsd and movss to avoid partial register stall
3343 // when loading from memory. But for old Opteron use movlpd instead of movsd.
3344 // The selection is done in MacroAssembler::movdbl() and movflt().
3345 void Assembler::movlpd(XMMRegister dst, Address src) {
3346 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3347 InstructionMark im(this);
3348 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3349 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3350 attributes.set_rex_vex_w_reverted();
3351 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3352 emit_int8(0x12);
3353 emit_operand(dst, src);
3354 }
3355
3356 void Assembler::movq(XMMRegister dst, Address src) {
3357 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3358 InstructionMark im(this);
3359 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3360 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3361 attributes.set_rex_vex_w_reverted();
3362 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3363 emit_int8(0x7E);
3364 emit_operand(dst, src);
3365 }
3366
3367 void Assembler::movq(Address dst, XMMRegister src) {
3368 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3369 InstructionMark im(this);
3370 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3371 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3372 attributes.set_rex_vex_w_reverted();
3373 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3374 emit_int8((unsigned char)0xD6);
3375 emit_operand(src, dst);
3376 }
3377
3378 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3379 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3380 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3381 attributes.set_rex_vex_w_reverted();
3382 int encode = simd_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3383 emit_int16((unsigned char)0xD6, (0xC0 | encode));
3384 }
3385
3386 void Assembler::movq(Register dst, XMMRegister src) {
3387 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3388 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3389 // swap src/dst to get correct prefix
3390 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3391 emit_int16(0x7E, (0xC0 | encode));
3392 }
3393
3394 void Assembler::movq(XMMRegister dst, Register src) {
3395 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3396 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3397 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3398 emit_int16(0x6E, (0xC0 | encode));
3399 }
3400
3401 void Assembler::movsbl(Register dst, Address src) { // movsxb
3402 InstructionMark im(this);
3403 prefix(src, dst);
3404 emit_int16(0x0F, (unsigned char)0xBE);
3405 emit_operand(dst, src);
3406 }
3407
3408 void Assembler::movsbl(Register dst, Register src) { // movsxb
3409 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3410 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3411 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
3412 }
3413
3414 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3415 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3416 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3417 attributes.set_rex_vex_w_reverted();
3418 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3419 emit_int16(0x10, (0xC0 | encode));
3420 }
3421
3422 void Assembler::movsd(XMMRegister dst, Address src) {
3423 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3424 InstructionMark im(this);
3425 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3426 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3427 attributes.set_rex_vex_w_reverted();
3428 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3429 emit_int8(0x10);
3430 emit_operand(dst, src);
3431 }
3432
3433 void Assembler::movsd(Address dst, XMMRegister src) {
3434 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3435 InstructionMark im(this);
3436 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3437 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3438 attributes.reset_is_clear_context();
3439 attributes.set_rex_vex_w_reverted();
3440 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3441 emit_int8(0x11);
3442 emit_operand(src, dst);
3443 }
3444
3445 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3446 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3447 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3448 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3449 emit_int16(0x10, (0xC0 | encode));
3450 }
3451
3452 void Assembler::movss(XMMRegister dst, Address src) {
3453 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3454 InstructionMark im(this);
3455 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3456 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3457 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3458 emit_int8(0x10);
3459 emit_operand(dst, src);
3460 }
3461
3462 void Assembler::movss(Address dst, XMMRegister src) {
3463 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3464 InstructionMark im(this);
3465 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3466 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3467 attributes.reset_is_clear_context();
3468 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3469 emit_int8(0x11);
3470 emit_operand(src, dst);
3471 }
3472
3473 void Assembler::movswl(Register dst, Address src) { // movsxw
3474 InstructionMark im(this);
3475 prefix(src, dst);
3476 emit_int16(0x0F, (unsigned char)0xBF);
3477 emit_operand(dst, src);
3478 }
3479
3480 void Assembler::movswl(Register dst, Register src) { // movsxw
3481 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3482 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
3483 }
3484
3485 void Assembler::movw(Address dst, int imm16) {
3486 InstructionMark im(this);
3487
3488 emit_int8(0x66); // switch to 16-bit mode
3489 prefix(dst);
3490 emit_int8((unsigned char)0xC7);
3491 emit_operand(rax, dst, 2);
3492 emit_int16(imm16);
3493 }
3494
3495 void Assembler::movw(Register dst, Address src) {
3496 InstructionMark im(this);
3497 emit_int8(0x66);
3498 prefix(src, dst);
3499 emit_int8((unsigned char)0x8B);
3500 emit_operand(dst, src);
3501 }
3502
3503 void Assembler::movw(Address dst, Register src) {
3504 InstructionMark im(this);
3505 emit_int8(0x66);
3506 prefix(dst, src);
3507 emit_int8((unsigned char)0x89);
3508 emit_operand(src, dst);
3509 }
3510
3511 void Assembler::movzbl(Register dst, Address src) { // movzxb
3512 InstructionMark im(this);
3513 prefix(src, dst);
3514 emit_int16(0x0F, (unsigned char)0xB6);
3515 emit_operand(dst, src);
3516 }
3517
3518 void Assembler::movzbl(Register dst, Register src) { // movzxb
3519 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
3520 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
3521 emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
3522 }
3523
3524 void Assembler::movzwl(Register dst, Address src) { // movzxw
3525 InstructionMark im(this);
3526 prefix(src, dst);
3527 emit_int16(0x0F, (unsigned char)0xB7);
3528 emit_operand(dst, src);
3529 }
3530
3531 void Assembler::movzwl(Register dst, Register src) { // movzxw
3532 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3533 emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
3534 }
3535
3536 void Assembler::mull(Address src) {
3537 InstructionMark im(this);
3538 prefix(src);
3539 emit_int8((unsigned char)0xF7);
3540 emit_operand(rsp, src);
3541 }
3542
3543 void Assembler::mull(Register src) {
3544 int encode = prefix_and_encode(src->encoding());
3545 emit_int16((unsigned char)0xF7, (0xE0 | encode));
3546 }
3547
3548 void Assembler::mulsd(XMMRegister dst, Address src) {
3549 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3550 InstructionMark im(this);
3551 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3552 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3553 attributes.set_rex_vex_w_reverted();
3554 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3555 emit_int8(0x59);
3556 emit_operand(dst, src);
3557 }
3558
3559 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3560 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3561 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3562 attributes.set_rex_vex_w_reverted();
3563 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3564 emit_int16(0x59, (0xC0 | encode));
3565 }
3566
3567 void Assembler::mulss(XMMRegister dst, Address src) {
3568 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3569 InstructionMark im(this);
3570 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3571 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3572 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3573 emit_int8(0x59);
3574 emit_operand(dst, src);
3575 }
3576
3577 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3578 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3579 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3580 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3581 emit_int16(0x59, (0xC0 | encode));
3582 }
3583
3584 void Assembler::negl(Register dst) {
3585 int encode = prefix_and_encode(dst->encoding());
3586 emit_int16((unsigned char)0xF7, (0xD8 | encode));
3587 }
3588
3589 void Assembler::negl(Address dst) {
3590 InstructionMark im(this);
3591 prefix(dst);
3592 emit_int8((unsigned char)0xF7);
3593 emit_operand(as_Register(3), dst);
3594 }
3595
3596 void Assembler::nop(int i) {
3597 #ifdef ASSERT
3598 assert(i > 0, " ");
3599 // The fancy nops aren't currently recognized by debuggers making it a
3600 // pain to disassemble code while debugging. If asserts are on clearly
3601 // speed is not an issue so simply use the single byte traditional nop
3602 // to do alignment.
3603
3604 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3605 return;
3606
3607 #endif // ASSERT
3608
3609 if (UseAddressNop && VM_Version::is_intel()) {
3610 //
3611 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3612 // 1: 0x90
3613 // 2: 0x66 0x90
3614 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3615 // 4: 0x0F 0x1F 0x40 0x00
3616 // 5: 0x0F 0x1F 0x44 0x00 0x00
3617 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3618 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3619 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3620 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3621 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3622 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3623
3624 // The rest coding is Intel specific - don't use consecutive address nops
3625
3626 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3627 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3628 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3629 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3630
3631 while(i >= 15) {
3632 // For Intel don't generate consecutive address nops (mix with regular nops)
3633 i -= 15;
3634 emit_int24(0x66, 0x66, 0x66);
3635 addr_nop_8();
3636 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3637 }
3638 switch (i) {
3639 case 14:
3640 emit_int8(0x66); // size prefix
3641 case 13:
3642 emit_int8(0x66); // size prefix
3643 case 12:
3644 addr_nop_8();
3645 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3646 break;
3647 case 11:
3648 emit_int8(0x66); // size prefix
3649 case 10:
3650 emit_int8(0x66); // size prefix
3651 case 9:
3652 emit_int8(0x66); // size prefix
3653 case 8:
3654 addr_nop_8();
3655 break;
3656 case 7:
3657 addr_nop_7();
3658 break;
3659 case 6:
3660 emit_int8(0x66); // size prefix
3661 case 5:
3662 addr_nop_5();
3663 break;
3664 case 4:
3665 addr_nop_4();
3666 break;
3667 case 3:
3668 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3669 emit_int8(0x66); // size prefix
3670 case 2:
3671 emit_int8(0x66); // size prefix
3672 case 1:
3673 emit_int8((unsigned char)0x90);
3674 // nop
3675 break;
3676 default:
3677 assert(i == 0, " ");
3678 }
3679 return;
3680 }
3681 if (UseAddressNop && VM_Version::is_amd_family()) {
3682 //
3683 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3684 // 1: 0x90
3685 // 2: 0x66 0x90
3686 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3687 // 4: 0x0F 0x1F 0x40 0x00
3688 // 5: 0x0F 0x1F 0x44 0x00 0x00
3689 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3690 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3691 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3692 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3693 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3694 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3695
3696 // The rest coding is AMD specific - use consecutive address nops
3697
3698 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3699 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3700 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3701 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3702 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3703 // Size prefixes (0x66) are added for larger sizes
3704
3705 while(i >= 22) {
3706 i -= 11;
3707 emit_int24(0x66, 0x66, 0x66);
3708 addr_nop_8();
3709 }
3710 // Generate first nop for size between 21-12
3711 switch (i) {
3712 case 21:
3713 i -= 1;
3714 emit_int8(0x66); // size prefix
3715 case 20:
3716 case 19:
3717 i -= 1;
3718 emit_int8(0x66); // size prefix
3719 case 18:
3720 case 17:
3721 i -= 1;
3722 emit_int8(0x66); // size prefix
3723 case 16:
3724 case 15:
3725 i -= 8;
3726 addr_nop_8();
3727 break;
3728 case 14:
3729 case 13:
3730 i -= 7;
3731 addr_nop_7();
3732 break;
3733 case 12:
3734 i -= 6;
3735 emit_int8(0x66); // size prefix
3736 addr_nop_5();
3737 break;
3738 default:
3739 assert(i < 12, " ");
3740 }
3741
3742 // Generate second nop for size between 11-1
3743 switch (i) {
3744 case 11:
3745 emit_int8(0x66); // size prefix
3746 case 10:
3747 emit_int8(0x66); // size prefix
3748 case 9:
3749 emit_int8(0x66); // size prefix
3750 case 8:
3751 addr_nop_8();
3752 break;
3753 case 7:
3754 addr_nop_7();
3755 break;
3756 case 6:
3757 emit_int8(0x66); // size prefix
3758 case 5:
3759 addr_nop_5();
3760 break;
3761 case 4:
3762 addr_nop_4();
3763 break;
3764 case 3:
3765 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3766 emit_int8(0x66); // size prefix
3767 case 2:
3768 emit_int8(0x66); // size prefix
3769 case 1:
3770 emit_int8((unsigned char)0x90);
3771 // nop
3772 break;
3773 default:
3774 assert(i == 0, " ");
3775 }
3776 return;
3777 }
3778
3779 if (UseAddressNop && VM_Version::is_zx()) {
3780 //
3781 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3782 // 1: 0x90
3783 // 2: 0x66 0x90
3784 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3785 // 4: 0x0F 0x1F 0x40 0x00
3786 // 5: 0x0F 0x1F 0x44 0x00 0x00
3787 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3788 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3789 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3790 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3791 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3792 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3793
3794 // The rest coding is ZX specific - don't use consecutive address nops
3795
3796 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3797 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3798 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3799 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3800
3801 while (i >= 15) {
3802 // For ZX don't generate consecutive address nops (mix with regular nops)
3803 i -= 15;
3804 emit_int24(0x66, 0x66, 0x66);
3805 addr_nop_8();
3806 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3807 }
3808 switch (i) {
3809 case 14:
3810 emit_int8(0x66); // size prefix
3811 case 13:
3812 emit_int8(0x66); // size prefix
3813 case 12:
3814 addr_nop_8();
3815 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3816 break;
3817 case 11:
3818 emit_int8(0x66); // size prefix
3819 case 10:
3820 emit_int8(0x66); // size prefix
3821 case 9:
3822 emit_int8(0x66); // size prefix
3823 case 8:
3824 addr_nop_8();
3825 break;
3826 case 7:
3827 addr_nop_7();
3828 break;
3829 case 6:
3830 emit_int8(0x66); // size prefix
3831 case 5:
3832 addr_nop_5();
3833 break;
3834 case 4:
3835 addr_nop_4();
3836 break;
3837 case 3:
3838 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3839 emit_int8(0x66); // size prefix
3840 case 2:
3841 emit_int8(0x66); // size prefix
3842 case 1:
3843 emit_int8((unsigned char)0x90);
3844 // nop
3845 break;
3846 default:
3847 assert(i == 0, " ");
3848 }
3849 return;
3850 }
3851
3852 // Using nops with size prefixes "0x66 0x90".
3853 // From AMD Optimization Guide:
3854 // 1: 0x90
3855 // 2: 0x66 0x90
3856 // 3: 0x66 0x66 0x90
3857 // 4: 0x66 0x66 0x66 0x90
3858 // 5: 0x66 0x66 0x90 0x66 0x90
3859 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
3860 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3861 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3862 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3863 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3864 //
3865 while (i > 12) {
3866 i -= 4;
3867 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3868 }
3869 // 1 - 12 nops
3870 if (i > 8) {
3871 if (i > 9) {
3872 i -= 1;
3873 emit_int8(0x66);
3874 }
3875 i -= 3;
3876 emit_int24(0x66, 0x66, (unsigned char)0x90);
3877 }
3878 // 1 - 8 nops
3879 if (i > 4) {
3880 if (i > 6) {
3881 i -= 1;
3882 emit_int8(0x66);
3883 }
3884 i -= 3;
3885 emit_int24(0x66, 0x66, (unsigned char)0x90);
3886 }
3887 switch (i) {
3888 case 4:
3889 emit_int8(0x66);
3890 case 3:
3891 emit_int8(0x66);
3892 case 2:
3893 emit_int8(0x66);
3894 case 1:
3895 emit_int8((unsigned char)0x90);
3896 break;
3897 default:
3898 assert(i == 0, " ");
3899 }
3900 }
3901
3902 void Assembler::notl(Register dst) {
3903 int encode = prefix_and_encode(dst->encoding());
3904 emit_int16((unsigned char)0xF7, (0xD0 | encode));
3905 }
3906
3907 void Assembler::orw(Register dst, Register src) {
3908 (void)prefix_and_encode(dst->encoding(), src->encoding());
3909 emit_arith(0x0B, 0xC0, dst, src);
3910 }
3911
3912 void Assembler::orl(Address dst, int32_t imm32) {
3913 InstructionMark im(this);
3914 prefix(dst);
3915 emit_arith_operand(0x81, rcx, dst, imm32);
3916 }
3917
3918 void Assembler::orl(Register dst, int32_t imm32) {
3919 prefix(dst);
3920 emit_arith(0x81, 0xC8, dst, imm32);
3921 }
3922
3923 void Assembler::orl(Register dst, Address src) {
3924 InstructionMark im(this);
3925 prefix(src, dst);
3926 emit_int8(0x0B);
3927 emit_operand(dst, src);
3928 }
3929
3930 void Assembler::orl(Register dst, Register src) {
3931 (void) prefix_and_encode(dst->encoding(), src->encoding());
3932 emit_arith(0x0B, 0xC0, dst, src);
3933 }
3934
3935 void Assembler::orl(Address dst, Register src) {
3936 InstructionMark im(this);
3937 prefix(dst, src);
3938 emit_int8(0x09);
3939 emit_operand(src, dst);
3940 }
3941
3942 void Assembler::orb(Address dst, int imm8) {
3943 InstructionMark im(this);
3944 prefix(dst);
3945 emit_int8((unsigned char)0x80);
3946 emit_operand(rcx, dst, 1);
3947 emit_int8(imm8);
3948 }
3949
3950 void Assembler::orb(Address dst, Register src) {
3951 InstructionMark im(this);
3952 prefix(dst, src, true);
3953 emit_int8(0x08);
3954 emit_operand(src, dst);
3955 }
3956
3957 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
3958 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3959 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3960 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3961 emit_int16(0x63, (0xC0 | encode));
3962 }
3963
3964 void Assembler::vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3965 assert(UseAVX > 0, "some form of AVX must be enabled");
3966 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3967 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3968 emit_int16(0x63, (0xC0 | encode));
3969 }
3970
3971 void Assembler::packssdw(XMMRegister dst, XMMRegister src) {
3972 assert(VM_Version::supports_sse2(), "");
3973 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3974 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3975 emit_int16(0x6B, (0xC0 | encode));
3976 }
3977
3978 void Assembler::vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3979 assert(UseAVX > 0, "some form of AVX must be enabled");
3980 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3981 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3982 emit_int16(0x6B, (0xC0 | encode));
3983 }
3984
3985 void Assembler::packuswb(XMMRegister dst, Address src) {
3986 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3987 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3988 InstructionMark im(this);
3989 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3990 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3991 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3992 emit_int8(0x67);
3993 emit_operand(dst, src);
3994 }
3995
3996 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3997 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3998 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3999 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4000 emit_int16(0x67, (0xC0 | encode));
4001 }
4002
4003 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4004 assert(UseAVX > 0, "some form of AVX must be enabled");
4005 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4006 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4007 emit_int16(0x67, (0xC0 | encode));
4008 }
4009
4010 void Assembler::packusdw(XMMRegister dst, XMMRegister src) {
4011 assert(VM_Version::supports_sse4_1(), "");
4012 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4013 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4014 emit_int16(0x2B, (0xC0 | encode));
4015 }
4016
4017 void Assembler::vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4018 assert(UseAVX > 0, "some form of AVX must be enabled");
4019 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4020 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4021 emit_int16(0x2B, (0xC0 | encode));
4022 }
4023
4024 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4025 assert(VM_Version::supports_avx2(), "");
4026 assert(vector_len != AVX_128bit, "");
4027 // VEX.256.66.0F3A.W1 00 /r ib
4028 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4029 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4030 emit_int24(0x00, (0xC0 | encode), imm8);
4031 }
4032
4033 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4034 assert(vector_len == AVX_256bit ? VM_Version::supports_avx512vl() :
4035 vector_len == AVX_512bit ? VM_Version::supports_evex() : false, "not supported");
4036 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4037 attributes.set_is_evex_instruction();
4038 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4039 emit_int16(0x36, (0xC0 | encode));
4040 }
4041
4042 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4043 assert(VM_Version::supports_avx512_vbmi(), "");
4044 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4045 attributes.set_is_evex_instruction();
4046 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4047 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4048 }
4049
4050 void Assembler::vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4051 assert(VM_Version::supports_avx512_vbmi(), "");
4052 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4053 attributes.set_is_evex_instruction();
4054 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4055 emit_int8((unsigned char)0x8D);
4056 emit_operand(dst, src);
4057 }
4058
4059 void Assembler::vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4060 assert(vector_len == AVX_128bit ? VM_Version::supports_avx512vlbw() :
4061 vector_len == AVX_256bit ? VM_Version::supports_avx512vlbw() :
4062 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : false, "not supported");
4063 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4064 attributes.set_is_evex_instruction();
4065 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4066 emit_int16((unsigned char)0x8D, (0xC0 | encode));
4067 }
4068
4069 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4070 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4071 // VEX.NDS.256.66.0F38.W0 36 /r
4072 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4073 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4074 emit_int16(0x36, (0xC0 | encode));
4075 }
4076
4077 void Assembler::vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4078 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4079 // VEX.NDS.256.66.0F38.W0 36 /r
4080 InstructionMark im(this);
4081 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4082 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4083 emit_int8(0x36);
4084 emit_operand(dst, src);
4085 }
4086
4087 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4088 assert(VM_Version::supports_avx2(), "");
4089 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4090 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4091 emit_int24(0x46, (0xC0 | encode), imm8);
4092 }
4093
4094 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4095 assert(VM_Version::supports_avx(), "");
4096 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4097 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4098 emit_int24(0x06, (0xC0 | encode), imm8);
4099 }
4100
4101 void Assembler::vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4102 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4103 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4104 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4105 emit_int24(0x04, (0xC0 | encode), imm8);
4106 }
4107
4108 void Assembler::vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4109 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
4110 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(),/* legacy_mode */ false,/* no_mask_reg */ true, /* uses_vl */ false);
4111 attributes.set_rex_vex_w_reverted();
4112 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4113 emit_int24(0x05, (0xC0 | encode), imm8);
4114 }
4115
4116 void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
4117 assert(vector_len <= AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex(), "");
4118 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */false, /* no_mask_reg */ true, /* uses_vl */ false);
4119 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4120 emit_int24(0x01, (0xC0 | encode), imm8);
4121 }
4122
4123 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4124 assert(VM_Version::supports_evex(), "");
4125 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4126 attributes.set_is_evex_instruction();
4127 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4128 emit_int16(0x76, (0xC0 | encode));
4129 }
4130
4131 void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4132 assert(VM_Version::supports_avx512_vbmi(), "");
4133 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4134 attributes.set_is_evex_instruction();
4135 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4136 emit_int16(0x7D, (0xC0 | encode));
4137 }
4138
4139 void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
4140 assert(VM_Version::supports_avx512_vbmi(), "");
4141 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4142 attributes.set_is_evex_instruction();
4143 int encode = vex_prefix_and_encode(dst->encoding(), ctl->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4144 emit_int16((unsigned char)0x83, (unsigned char)(0xC0 | encode));
4145 }
4146
4147 void Assembler::pause() {
4148 emit_int16((unsigned char)0xF3, (unsigned char)0x90);
4149 }
4150
4151 void Assembler::ud2() {
4152 emit_int16(0x0F, 0x0B);
4153 }
4154
4155 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
4156 assert(VM_Version::supports_sse4_2(), "");
4157 InstructionMark im(this);
4158 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4159 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4160 emit_int8(0x61);
4161 emit_operand(dst, src);
4162 emit_int8(imm8);
4163 }
4164
4165 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
4166 assert(VM_Version::supports_sse4_2(), "");
4167 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4168 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4169 emit_int24(0x61, (0xC0 | encode), imm8);
4170 }
4171
4172 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4173 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
4174 assert(VM_Version::supports_sse2(), "");
4175 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4176 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4177 emit_int16(0x74, (0xC0 | encode));
4178 }
4179
4180 void Assembler::vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4181 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4182 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4183 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4184 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4185 emit_int16(cond_encoding, (0xC0 | encode));
4186 }
4187
4188 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4189 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4190 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4191 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4192 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4193 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4194 emit_int16(0x74, (0xC0 | encode));
4195 }
4196
4197 // In this context, kdst is written the mask used to process the equal components
4198 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4199 assert(VM_Version::supports_avx512bw(), "");
4200 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4201 attributes.set_is_evex_instruction();
4202 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4203 emit_int16(0x74, (0xC0 | encode));
4204 }
4205
4206 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4207 assert(VM_Version::supports_avx512vlbw(), "");
4208 InstructionMark im(this);
4209 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4210 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4211 attributes.set_is_evex_instruction();
4212 int dst_enc = kdst->encoding();
4213 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4214 emit_int8(0x64);
4215 emit_operand(as_Register(dst_enc), src);
4216 }
4217
4218 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4219 assert(VM_Version::supports_avx512vlbw(), "");
4220 InstructionMark im(this);
4221 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4222 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4223 attributes.reset_is_clear_context();
4224 attributes.set_embedded_opmask_register_specifier(mask);
4225 attributes.set_is_evex_instruction();
4226 int dst_enc = kdst->encoding();
4227 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4228 emit_int8(0x64);
4229 emit_operand(as_Register(dst_enc), src);
4230 }
4231
4232 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
4233 assert(VM_Version::supports_avx512vlbw(), "");
4234 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4235 attributes.set_is_evex_instruction();
4236 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4237 emit_int24(0x3E, (0xC0 | encode), vcc);
4238 }
4239
4240 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
4241 assert(VM_Version::supports_avx512vlbw(), "");
4242 InstructionMark im(this);
4243 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4244 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4245 attributes.set_is_evex_instruction();
4246 int dst_enc = kdst->encoding();
4247 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4248 emit_int8(0x3E);
4249 emit_operand(as_Register(dst_enc), src);
4250 emit_int8(vcc);
4251 }
4252
4253 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4254 assert(VM_Version::supports_avx512bw(), "");
4255 InstructionMark im(this);
4256 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4257 attributes.set_is_evex_instruction();
4258 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4259 int dst_enc = kdst->encoding();
4260 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4261 emit_int8(0x74);
4262 emit_operand(as_Register(dst_enc), src);
4263 }
4264
4265 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4266 assert(VM_Version::supports_avx512vlbw(), "");
4267 InstructionMark im(this);
4268 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4269 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4270 attributes.reset_is_clear_context();
4271 attributes.set_embedded_opmask_register_specifier(mask);
4272 attributes.set_is_evex_instruction();
4273 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4274 emit_int8(0x74);
4275 emit_operand(as_Register(kdst->encoding()), src);
4276 }
4277
4278 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4279 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
4280 assert(VM_Version::supports_sse2(), "");
4281 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4282 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4283 emit_int16(0x75, (0xC0 | encode));
4284 }
4285
4286 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4287 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4288 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4289 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4290 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4291 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4292 emit_int16(0x75, (0xC0 | encode));
4293 }
4294
4295 // In this context, kdst is written the mask used to process the equal components
4296 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4297 assert(VM_Version::supports_avx512bw(), "");
4298 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4299 attributes.set_is_evex_instruction();
4300 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4301 emit_int16(0x75, (0xC0 | encode));
4302 }
4303
4304 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4305 assert(VM_Version::supports_avx512bw(), "");
4306 InstructionMark im(this);
4307 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4308 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4309 attributes.set_is_evex_instruction();
4310 int dst_enc = kdst->encoding();
4311 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4312 emit_int8(0x75);
4313 emit_operand(as_Register(dst_enc), src);
4314 }
4315
4316 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4317 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
4318 assert(VM_Version::supports_sse2(), "");
4319 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4320 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4321 emit_int16(0x76, (0xC0 | encode));
4322 }
4323
4324 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4325 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4326 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
4327 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
4328 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4329 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4330 emit_int16(0x76, (0xC0 | encode));
4331 }
4332
4333 // In this context, kdst is written the mask used to process the equal components
4334 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4335 assert(VM_Version::supports_evex(), "");
4336 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4337 attributes.set_is_evex_instruction();
4338 attributes.reset_is_clear_context();
4339 attributes.set_embedded_opmask_register_specifier(mask);
4340 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4341 emit_int16(0x76, (0xC0 | encode));
4342 }
4343
4344 void Assembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
4345 assert(VM_Version::supports_evex(), "");
4346 InstructionMark im(this);
4347 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4348 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4349 attributes.set_is_evex_instruction();
4350 attributes.reset_is_clear_context();
4351 attributes.set_embedded_opmask_register_specifier(mask);
4352 int dst_enc = kdst->encoding();
4353 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4354 emit_int8(0x76);
4355 emit_operand(as_Register(dst_enc), src);
4356 }
4357
4358 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4359 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
4360 assert(VM_Version::supports_sse4_1(), "");
4361 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4362 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4363 emit_int16(0x29, (0xC0 | encode));
4364 }
4365
4366 void Assembler::evpcmpeqq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len) {
4367 assert(VM_Version::supports_evex(), "");
4368 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4369 attributes.set_is_evex_instruction();
4370 attributes.reset_is_clear_context();
4371 attributes.set_embedded_opmask_register_specifier(mask);
4372 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4373 emit_int16(0x29, (0xC0 | encode));
4374 }
4375
4376 void Assembler::vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len) {
4377 assert(VM_Version::supports_avx(), "");
4378 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4379 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4380 emit_int16(cond_encoding, (0xC0 | encode));
4381 }
4382
4383 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
4384 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4385 assert(VM_Version::supports_avx(), "");
4386 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4387 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4388 emit_int16(0x29, (0xC0 | encode));
4389 }
4390
4391 // In this context, kdst is written the mask used to process the equal components
4392 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
4393 assert(VM_Version::supports_evex(), "");
4394 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4395 attributes.reset_is_clear_context();
4396 attributes.set_is_evex_instruction();
4397 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4398 emit_int16(0x29, (0xC0 | encode));
4399 }
4400
4401 // In this context, kdst is written the mask used to process the equal components
4402 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
4403 assert(VM_Version::supports_evex(), "");
4404 InstructionMark im(this);
4405 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4406 attributes.reset_is_clear_context();
4407 attributes.set_is_evex_instruction();
4408 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4409 int dst_enc = kdst->encoding();
4410 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4411 emit_int8(0x29);
4412 emit_operand(as_Register(dst_enc), src);
4413 }
4414
4415 void Assembler::pcmpgtq(XMMRegister dst, XMMRegister src) {
4416 assert(VM_Version::supports_sse4_1(), "");
4417 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4418 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4419 emit_int16(0x37, (0xC0 | encode));
4420 }
4421
4422 void Assembler::pmovmskb(Register dst, XMMRegister src) {
4423 assert(VM_Version::supports_sse2(), "");
4424 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4425 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4426 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4427 }
4428
4429 void Assembler::vpmovmskb(Register dst, XMMRegister src, int vec_enc) {
4430 assert((VM_Version::supports_avx() && vec_enc == AVX_128bit) ||
4431 (VM_Version::supports_avx2() && vec_enc == AVX_256bit), "");
4432 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4433 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4434 emit_int16((unsigned char)0xD7, (0xC0 | encode));
4435 }
4436
4437 void Assembler::vmovmskps(Register dst, XMMRegister src, int vec_enc) {
4438 assert(VM_Version::supports_avx(), "");
4439 InstructionAttr attributes(vec_enc, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4440 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4441 emit_int16(0x50, (0xC0 | encode));
4442 }
4443
4444 void Assembler::vmovmskpd(Register dst, XMMRegister src, int vec_enc) {
4445 assert(VM_Version::supports_avx(), "");
4446 InstructionAttr attributes(vec_enc, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4447 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4448 emit_int16(0x50, (0xC0 | encode));
4449 }
4450
4451
4452 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
4453 assert(VM_Version::supports_sse4_1(), "");
4454 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4455 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4456 emit_int24(0x16, (0xC0 | encode), imm8);
4457 }
4458
4459 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
4460 assert(VM_Version::supports_sse4_1(), "");
4461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4462 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4463 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4464 emit_int8(0x16);
4465 emit_operand(src, dst);
4466 emit_int8(imm8);
4467 }
4468
4469 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
4470 assert(VM_Version::supports_sse4_1(), "");
4471 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4472 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4473 emit_int24(0x16, (0xC0 | encode), imm8);
4474 }
4475
4476 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
4477 assert(VM_Version::supports_sse4_1(), "");
4478 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4479 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4480 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4481 emit_int8(0x16);
4482 emit_operand(src, dst);
4483 emit_int8(imm8);
4484 }
4485
4486 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
4487 assert(VM_Version::supports_sse2(), "");
4488 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4489 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4490 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
4491 }
4492
4493 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
4494 assert(VM_Version::supports_sse4_1(), "");
4495 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4496 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4497 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4498 emit_int8(0x15);
4499 emit_operand(src, dst);
4500 emit_int8(imm8);
4501 }
4502
4503 void Assembler::pextrb(Register dst, XMMRegister src, int imm8) {
4504 assert(VM_Version::supports_sse4_1(), "");
4505 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4506 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4507 emit_int24(0x14, (0xC0 | encode), imm8);
4508 }
4509
4510 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
4511 assert(VM_Version::supports_sse4_1(), "");
4512 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4513 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4514 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4515 emit_int8(0x14);
4516 emit_operand(src, dst);
4517 emit_int8(imm8);
4518 }
4519
4520 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
4521 assert(VM_Version::supports_sse4_1(), "");
4522 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4523 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4524 emit_int24(0x22, (0xC0 | encode), imm8);
4525 }
4526
4527 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
4528 assert(VM_Version::supports_sse4_1(), "");
4529 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4530 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4531 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4532 emit_int8(0x22);
4533 emit_operand(dst,src);
4534 emit_int8(imm8);
4535 }
4536
4537 void Assembler::vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4538 assert(VM_Version::supports_avx(), "");
4539 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4540 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4541 emit_int24(0x22, (0xC0 | encode), imm8);
4542 }
4543
4544 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
4545 assert(VM_Version::supports_sse4_1(), "");
4546 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4547 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4548 emit_int24(0x22, (0xC0 | encode), imm8);
4549 }
4550
4551 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
4552 assert(VM_Version::supports_sse4_1(), "");
4553 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4554 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4555 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4556 emit_int8(0x22);
4557 emit_operand(dst, src);
4558 emit_int8(imm8);
4559 }
4560
4561 void Assembler::vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4562 assert(VM_Version::supports_avx(), "");
4563 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
4564 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4565 emit_int24(0x22, (0xC0 | encode), imm8);
4566 }
4567
4568 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
4569 assert(VM_Version::supports_sse2(), "");
4570 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4571 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4572 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4573 }
4574
4575 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
4576 assert(VM_Version::supports_sse2(), "");
4577 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4578 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
4579 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4580 emit_int8((unsigned char)0xC4);
4581 emit_operand(dst, src);
4582 emit_int8(imm8);
4583 }
4584
4585 void Assembler::vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4586 assert(VM_Version::supports_avx(), "");
4587 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4588 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4589 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
4590 }
4591
4592 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
4593 assert(VM_Version::supports_sse4_1(), "");
4594 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4595 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
4596 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4597 emit_int8(0x20);
4598 emit_operand(dst, src);
4599 emit_int8(imm8);
4600 }
4601
4602 void Assembler::pinsrb(XMMRegister dst, Register src, int imm8) {
4603 assert(VM_Version::supports_sse4_1(), "");
4604 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4605 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4606 emit_int24(0x20, (0xC0 | encode), imm8);
4607 }
4608
4609 void Assembler::vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8) {
4610 assert(VM_Version::supports_avx(), "");
4611 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4612 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4613 emit_int24(0x20, (0xC0 | encode), imm8);
4614 }
4615
4616 void Assembler::insertps(XMMRegister dst, XMMRegister src, int imm8) {
4617 assert(VM_Version::supports_sse4_1(), "");
4618 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4619 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4620 emit_int24(0x21, (0xC0 | encode), imm8);
4621 }
4622
4623 void Assembler::vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
4624 assert(VM_Version::supports_avx(), "");
4625 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4626 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4627 emit_int24(0x21, (0xC0 | encode), imm8);
4628 }
4629
4630 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
4631 assert(VM_Version::supports_sse4_1(), "");
4632 InstructionMark im(this);
4633 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4634 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4635 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4636 emit_int8(0x30);
4637 emit_operand(dst, src);
4638 }
4639
4640 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
4641 assert(VM_Version::supports_sse4_1(), "");
4642 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4643 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4644 emit_int16(0x30, (0xC0 | encode));
4645 }
4646
4647 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
4648 assert(VM_Version::supports_sse4_1(), "");
4649 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4650 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4651 emit_int16(0x20, (0xC0 | encode));
4652 }
4653
4654 void Assembler::pmovzxdq(XMMRegister dst, XMMRegister src) {
4655 assert(VM_Version::supports_sse4_1(), "");
4656 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4657 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4658 emit_int16(0x35, (0xC0 | encode));
4659 }
4660
4661 void Assembler::pmovsxbd(XMMRegister dst, XMMRegister src) {
4662 assert(VM_Version::supports_sse4_1(), "");
4663 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4664 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4665 emit_int16(0x21, (0xC0 | encode));
4666 }
4667
4668 void Assembler::pmovzxbd(XMMRegister dst, XMMRegister src) {
4669 assert(VM_Version::supports_sse4_1(), "");
4670 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4671 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4672 emit_int16(0x31, (0xC0 | encode));
4673 }
4674
4675 void Assembler::pmovsxbq(XMMRegister dst, XMMRegister src) {
4676 assert(VM_Version::supports_sse4_1(), "");
4677 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4678 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4679 emit_int16(0x22, (0xC0 | encode));
4680 }
4681
4682 void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
4683 assert(VM_Version::supports_sse4_1(), "");
4684 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4685 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4686 emit_int16(0x23, (0xC0 | encode));
4687 }
4688
4689 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4690 assert(VM_Version::supports_avx(), "");
4691 InstructionMark im(this);
4692 assert(dst != xnoreg, "sanity");
4693 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4694 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4695 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4696 emit_int8(0x30);
4697 emit_operand(dst, src);
4698 }
4699
4700 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4701 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4702 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4703 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4704 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4705 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4706 emit_int16(0x30, (unsigned char) (0xC0 | encode));
4707 }
4708
4709 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
4710 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4711 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4712 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4713 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4714 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4715 emit_int16(0x20, (0xC0 | encode));
4716 }
4717
4718 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
4719 assert(VM_Version::supports_avx512vlbw(), "");
4720 assert(dst != xnoreg, "sanity");
4721 InstructionMark im(this);
4722 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4723 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4724 attributes.set_embedded_opmask_register_specifier(mask);
4725 attributes.set_is_evex_instruction();
4726 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4727 emit_int8(0x30);
4728 emit_operand(dst, src);
4729 }
4730
4731 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
4732 assert(VM_Version::supports_evex(), "");
4733 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
4734 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4735 attributes.set_is_evex_instruction();
4736 attributes.set_embedded_opmask_register_specifier(mask);
4737 if (merge) {
4738 attributes.reset_is_clear_context();
4739 }
4740 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4741 emit_int16((unsigned char)0xDB, (0xC0 | encode));
4742 }
4743
4744 void Assembler::vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4745 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4746 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4747 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4748 emit_int16(0x35, (0xC0 | encode));
4749 }
4750
4751 void Assembler::vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4752 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4753 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4754 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4755 emit_int16(0x31, (0xC0 | encode));
4756 }
4757
4758 void Assembler::vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4759 assert(vector_len > AVX_128bit ? VM_Version::supports_avx2() : VM_Version::supports_avx(), "");
4760 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4761 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4762 emit_int16(0x32, (0xC0 | encode));
4763 }
4764
4765 void Assembler::vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len) {
4766 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4767 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4768 VM_Version::supports_evex(), "");
4769 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4770 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4771 emit_int16(0x21, (0xC0 | encode));
4772 }
4773
4774 void Assembler::vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len) {
4775 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4776 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4777 VM_Version::supports_evex(), "");
4778 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4779 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4780 emit_int16(0x22, (0xC0 | encode));
4781 }
4782
4783 void Assembler::vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len) {
4784 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4785 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4786 VM_Version::supports_evex(), "");
4787 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4788 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4789 emit_int16(0x23, (0xC0 | encode));
4790 }
4791
4792 void Assembler::vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4793 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4794 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4795 VM_Version::supports_evex(), "");
4796 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4797 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4798 emit_int16(0x24, (0xC0 | encode));
4799 }
4800
4801 void Assembler::vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len) {
4802 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4803 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4804 VM_Version::supports_evex(), "");
4805 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4806 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4807 emit_int16(0x25, (0xC0 | encode));
4808 }
4809
4810 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
4811 assert(VM_Version::supports_avx512vlbw(), "");
4812 assert(src != xnoreg, "sanity");
4813 InstructionMark im(this);
4814 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4815 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4816 attributes.set_is_evex_instruction();
4817 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4818 emit_int8(0x30);
4819 emit_operand(src, dst);
4820 }
4821
4822 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
4823 assert(VM_Version::supports_avx512vlbw(), "");
4824 assert(src != xnoreg, "sanity");
4825 InstructionMark im(this);
4826 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4827 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
4828 attributes.reset_is_clear_context();
4829 attributes.set_embedded_opmask_register_specifier(mask);
4830 attributes.set_is_evex_instruction();
4831 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4832 emit_int8(0x30);
4833 emit_operand(src, dst);
4834 }
4835
4836 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
4837 assert(VM_Version::supports_evex(), "");
4838 assert(src != xnoreg, "sanity");
4839 InstructionMark im(this);
4840 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4841 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
4842 attributes.set_is_evex_instruction();
4843 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
4844 emit_int8(0x31);
4845 emit_operand(src, dst);
4846 }
4847
4848 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
4849 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4850 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4851 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4852 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4853 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4854 emit_int16(0x33, (0xC0 | encode));
4855 }
4856
4857 void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4858 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4859 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4860 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4861 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4862 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4863 emit_int16(0x34, (0xC0 | encode));
4864 }
4865
4866 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
4867 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4868 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4869 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4870 emit_int16((unsigned char)0xF5, (0xC0 | encode));
4871 }
4872
4873 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4874 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4875 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4876 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
4877 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4878 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4879 emit_int16((unsigned char)0xF5, (0xC0 | encode));
4880 }
4881
4882 void Assembler::vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
4883 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4884 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4885 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4886 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4887 int encode = simd_prefix_and_encode(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4888 emit_int16(0x04, (0xC0 | encode));
4889 }
4890
4891 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4892 assert(VM_Version::supports_evex(), "");
4893 assert(VM_Version::supports_avx512_vnni(), "must support vnni");
4894 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4895 attributes.set_is_evex_instruction();
4896 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4897 emit_int16(0x52, (0xC0 | encode));
4898 }
4899
4900 // generic
4901 void Assembler::pop(Register dst) {
4902 int encode = prefix_and_encode(dst->encoding());
4903 emit_int8(0x58 | encode);
4904 }
4905
4906 void Assembler::popcntl(Register dst, Address src) {
4907 assert(VM_Version::supports_popcnt(), "must support");
4908 InstructionMark im(this);
4909 emit_int8((unsigned char)0xF3);
4910 prefix(src, dst);
4911 emit_int16(0x0F, (unsigned char)0xB8);
4912 emit_operand(dst, src);
4913 }
4914
4915 void Assembler::popcntl(Register dst, Register src) {
4916 assert(VM_Version::supports_popcnt(), "must support");
4917 emit_int8((unsigned char)0xF3);
4918 int encode = prefix_and_encode(dst->encoding(), src->encoding());
4919 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
4920 }
4921
4922 void Assembler::evpopcntb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
4923 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
4924 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
4925 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4926 attributes.set_embedded_opmask_register_specifier(mask);
4927 attributes.set_is_evex_instruction();
4928 if (merge) {
4929 attributes.reset_is_clear_context();
4930 }
4931 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4932 emit_int16(0x54, (0xC0 | encode));
4933 }
4934
4935 void Assembler::evpopcntw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
4936 assert(VM_Version::supports_avx512_bitalg(), "must support avx512bitalg feature");
4937 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
4938 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4939 attributes.set_is_evex_instruction();
4940 attributes.set_embedded_opmask_register_specifier(mask);
4941 if (merge) {
4942 attributes.reset_is_clear_context();
4943 }
4944 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4945 emit_int16(0x54, (0xC0 | encode));
4946 }
4947
4948 void Assembler::evpopcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
4949 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
4950 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
4951 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4952 attributes.set_is_evex_instruction();
4953 attributes.set_embedded_opmask_register_specifier(mask);
4954 if (merge) {
4955 attributes.reset_is_clear_context();
4956 }
4957 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4958 emit_int16(0x55, (0xC0 | encode));
4959 }
4960
4961 void Assembler::evpopcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
4962 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
4963 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
4964 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4965 attributes.set_is_evex_instruction();
4966 attributes.set_embedded_opmask_register_specifier(mask);
4967 if (merge) {
4968 attributes.reset_is_clear_context();
4969 }
4970 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4971 emit_int16(0x55, (0xC0 | encode));
4972 }
4973
4974 void Assembler::popf() {
4975 emit_int8((unsigned char)0x9D);
4976 }
4977
4978 #ifndef _LP64 // no 32bit push/pop on amd64
4979 void Assembler::popl(Address dst) {
4980 // NOTE: this will adjust stack by 8byte on 64bits
4981 InstructionMark im(this);
4982 prefix(dst);
4983 emit_int8((unsigned char)0x8F);
4984 emit_operand(rax, dst);
4985 }
4986 #endif
4987
4988 void Assembler::prefetchnta(Address src) {
4989 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4990 InstructionMark im(this);
4991 prefix(src);
4992 emit_int16(0x0F, 0x18);
4993 emit_operand(rax, src); // 0, src
4994 }
4995
4996 void Assembler::prefetchr(Address src) {
4997 assert(VM_Version::supports_3dnow_prefetch(), "must support");
4998 InstructionMark im(this);
4999 prefix(src);
5000 emit_int16(0x0F, 0x0D);
5001 emit_operand(rax, src); // 0, src
5002 }
5003
5004 void Assembler::prefetcht0(Address src) {
5005 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5006 InstructionMark im(this);
5007 prefix(src);
5008 emit_int16(0x0F, 0x18);
5009 emit_operand(rcx, src); // 1, src
5010 }
5011
5012 void Assembler::prefetcht1(Address src) {
5013 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5014 InstructionMark im(this);
5015 prefix(src);
5016 emit_int16(0x0F, 0x18);
5017 emit_operand(rdx, src); // 2, src
5018 }
5019
5020 void Assembler::prefetcht2(Address src) {
5021 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
5022 InstructionMark im(this);
5023 prefix(src);
5024 emit_int16(0x0F, 0x18);
5025 emit_operand(rbx, src); // 3, src
5026 }
5027
5028 void Assembler::prefetchw(Address src) {
5029 assert(VM_Version::supports_3dnow_prefetch(), "must support");
5030 InstructionMark im(this);
5031 prefix(src);
5032 emit_int16(0x0F, 0x0D);
5033 emit_operand(rcx, src); // 1, src
5034 }
5035
5036 void Assembler::prefix(Prefix p) {
5037 emit_int8(p);
5038 }
5039
5040 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
5041 assert(VM_Version::supports_ssse3(), "");
5042 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5043 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5044 emit_int16(0x00, (0xC0 | encode));
5045 }
5046
5047 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5048 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5049 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5050 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
5051 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5052 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5053 emit_int16(0x00, (0xC0 | encode));
5054 }
5055
5056 void Assembler::pshufb(XMMRegister dst, Address src) {
5057 assert(VM_Version::supports_ssse3(), "");
5058 InstructionMark im(this);
5059 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5060 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5061 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5062 emit_int8(0x00);
5063 emit_operand(dst, src);
5064 }
5065
5066 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
5067 assert(isByte(mode), "invalid value");
5068 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5069 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
5070 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5071 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5072 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5073 }
5074
5075 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
5076 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5077 (vector_len == AVX_256bit? VM_Version::supports_avx2() :
5078 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
5079 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5080 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5081 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5082 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5083 }
5084
5085 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
5086 assert(isByte(mode), "invalid value");
5087 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5088 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5089 InstructionMark im(this);
5090 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5091 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5092 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5093 emit_int8(0x70);
5094 emit_operand(dst, src);
5095 emit_int8(mode & 0xFF);
5096 }
5097
5098 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, int mode) {
5099 assert(isByte(mode), "invalid value");
5100 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5101 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5102 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5103 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5104 }
5105
5106 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5107 assert(isByte(mode), "invalid value");
5108 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5109 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5110 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5111 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
5112 }
5113
5114 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
5115 assert(isByte(mode), "invalid value");
5116 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5117 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5118 InstructionMark im(this);
5119 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5120 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5121 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5122 emit_int8(0x70);
5123 emit_operand(dst, src);
5124 emit_int8(mode & 0xFF);
5125 }
5126
5127 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5128 assert(VM_Version::supports_evex(), "requires EVEX support");
5129 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
5130 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5131 attributes.set_is_evex_instruction();
5132 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5133 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
5134 }
5135
5136 void Assembler::pshufpd(XMMRegister dst, XMMRegister src, int imm8) {
5137 assert(isByte(imm8), "invalid value");
5138 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5139 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5140 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5141 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5142 }
5143
5144 void Assembler::vpshufpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5145 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5146 attributes.set_rex_vex_w_reverted();
5147 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5148 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5149 }
5150
5151 void Assembler::pshufps(XMMRegister dst, XMMRegister src, int imm8) {
5152 assert(isByte(imm8), "invalid value");
5153 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5154 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5155 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5156 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5157 }
5158
5159 void Assembler::vpshufps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5160 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5161 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5162 emit_int24((unsigned char)0xC6, (0xC0 | encode), imm8 & 0xFF);
5163 }
5164
5165 void Assembler::psrldq(XMMRegister dst, int shift) {
5166 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5167 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5168 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5169 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5170 emit_int24(0x73, (0xC0 | encode), shift);
5171 }
5172
5173 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5174 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5175 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5176 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5177 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5178 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5179 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5180 }
5181
5182 void Assembler::pslldq(XMMRegister dst, int shift) {
5183 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
5184 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5185 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5186 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
5187 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5188 emit_int24(0x73, (0xC0 | encode), shift);
5189 }
5190
5191 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5192 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
5193 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
5194 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
5195 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5196 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5197 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5198 }
5199
5200 void Assembler::ptest(XMMRegister dst, Address src) {
5201 assert(VM_Version::supports_sse4_1(), "");
5202 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5203 InstructionMark im(this);
5204 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5205 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5206 emit_int8(0x17);
5207 emit_operand(dst, src);
5208 }
5209
5210 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
5211 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
5212 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5213 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5214 emit_int8(0x17);
5215 emit_int8((0xC0 | encode));
5216 }
5217
5218 void Assembler::vptest(XMMRegister dst, Address src) {
5219 assert(VM_Version::supports_avx(), "");
5220 InstructionMark im(this);
5221 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5222 assert(dst != xnoreg, "sanity");
5223 // swap src<->dst for encoding
5224 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5225 emit_int8(0x17);
5226 emit_operand(dst, src);
5227 }
5228
5229 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
5230 assert(VM_Version::supports_avx(), "");
5231 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5232 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5233 emit_int16(0x17, (0xC0 | encode));
5234 }
5235
5236 void Assembler::vptest(XMMRegister dst, XMMRegister src, int vector_len) {
5237 assert(VM_Version::supports_avx(), "");
5238 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5239 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5240 emit_int16(0x17, (0xC0 | encode));
5241 }
5242
5243 void Assembler::evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5244 assert(VM_Version::supports_avx512vlbw(), "");
5245 // Encoding: EVEX.NDS.XXX.66.0F.W0 DB /r
5246 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5247 attributes.set_is_evex_instruction();
5248 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5249 emit_int16((unsigned char)0x26, (0xC0 | encode));
5250 }
5251
5252 void Assembler::punpcklbw(XMMRegister dst, Address src) {
5253 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5254 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5255 InstructionMark im(this);
5256 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5257 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5258 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5259 emit_int8(0x60);
5260 emit_operand(dst, src);
5261 }
5262
5263 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
5264 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5265 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
5266 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5267 emit_int16(0x60, (0xC0 | encode));
5268 }
5269
5270 void Assembler::punpckldq(XMMRegister dst, Address src) {
5271 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5272 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
5273 InstructionMark im(this);
5274 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5275 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5276 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5277 emit_int8(0x62);
5278 emit_operand(dst, src);
5279 }
5280
5281 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
5282 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5283 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5284 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5285 emit_int16(0x62, (0xC0 | encode));
5286 }
5287
5288 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
5289 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5290 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5291 attributes.set_rex_vex_w_reverted();
5292 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5293 emit_int16(0x6C, (0xC0 | encode));
5294 }
5295
5296 void Assembler::push(int32_t imm32) {
5297 // in 64bits we push 64bits onto the stack but only
5298 // take a 32bit immediate
5299 emit_int8(0x68);
5300 emit_int32(imm32);
5301 }
5302
5303 void Assembler::push(Register src) {
5304 int encode = prefix_and_encode(src->encoding());
5305 emit_int8(0x50 | encode);
5306 }
5307
5308 void Assembler::pushf() {
5309 emit_int8((unsigned char)0x9C);
5310 }
5311
5312 #ifndef _LP64 // no 32bit push/pop on amd64
5313 void Assembler::pushl(Address src) {
5314 // Note this will push 64bit on 64bit
5315 InstructionMark im(this);
5316 prefix(src);
5317 emit_int8((unsigned char)0xFF);
5318 emit_operand(rsi, src);
5319 }
5320 #endif
5321
5322 void Assembler::rcll(Register dst, int imm8) {
5323 assert(isShiftCount(imm8), "illegal shift count");
5324 int encode = prefix_and_encode(dst->encoding());
5325 if (imm8 == 1) {
5326 emit_int16((unsigned char)0xD1, (0xD0 | encode));
5327 } else {
5328 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
5329 }
5330 }
5331
5332 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
5333 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5334 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5335 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5336 emit_int16(0x53, (0xC0 | encode));
5337 }
5338
5339 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
5340 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5341 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5342 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5343 emit_int16(0x53, (0xC0 | encode));
5344 }
5345
5346 void Assembler::rdtsc() {
5347 emit_int16(0x0F, 0x31);
5348 }
5349
5350 void Assembler::rdtscp() {
5351 emit_int24(0x0F, 0x01, (unsigned char)0xF9);
5352 }
5353
5354 // copies data from [esi] to [edi] using rcx pointer sized words
5355 // generic
5356 void Assembler::rep_mov() {
5357 // REP
5358 // MOVSQ
5359 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
5360 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xA5);)
5361 }
5362
5363 // sets rcx bytes with rax, value at [edi]
5364 void Assembler::rep_stosb() {
5365 // REP
5366 // STOSB
5367 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
5368 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAA);)
5369 }
5370
5371 // sets rcx pointer sized words with rax, value at [edi]
5372 // generic
5373 void Assembler::rep_stos() {
5374 // REP
5375 // LP64:STOSQ, LP32:STOSD
5376 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
5377 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAB);)
5378 }
5379
5380 // scans rcx pointer sized words at [edi] for occurrence of rax,
5381 // generic
5382 void Assembler::repne_scan() { // repne_scan
5383 // SCASQ
5384 LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
5385 NOT_LP64( emit_int16((unsigned char)0xF2, (unsigned char)0xAF);)
5386 }
5387
5388 #ifdef _LP64
5389 // scans rcx 4 byte words at [edi] for occurrence of rax,
5390 // generic
5391 void Assembler::repne_scanl() { // repne_scan
5392 // SCASL
5393 emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
5394 }
5395 #endif
5396
5397 void Assembler::ret(int imm16) {
5398 if (imm16 == 0) {
5399 emit_int8((unsigned char)0xC3);
5400 } else {
5401 emit_int8((unsigned char)0xC2);
5402 emit_int16(imm16);
5403 }
5404 }
5405
5406 void Assembler::roll(Register dst, int imm8) {
5407 assert(isShiftCount(imm8), "illegal shift count");
5408 int encode = prefix_and_encode(dst->encoding());
5409 if (imm8 == 1) {
5410 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5411 } else {
5412 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5413 }
5414 }
5415
5416 void Assembler::roll(Register dst) {
5417 int encode = prefix_and_encode(dst->encoding());
5418 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5419 }
5420
5421 void Assembler::rorl(Register dst, int imm8) {
5422 assert(isShiftCount(imm8), "illegal shift count");
5423 int encode = prefix_and_encode(dst->encoding());
5424 if (imm8 == 1) {
5425 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5426 } else {
5427 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5428 }
5429 }
5430
5431 void Assembler::rorl(Register dst) {
5432 int encode = prefix_and_encode(dst->encoding());
5433 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5434 }
5435
5436 #ifdef _LP64
5437 void Assembler::rorq(Register dst) {
5438 int encode = prefixq_and_encode(dst->encoding());
5439 emit_int16((unsigned char)0xD3, (0xC8 | encode));
5440 }
5441
5442 void Assembler::rorq(Register dst, int imm8) {
5443 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5444 int encode = prefixq_and_encode(dst->encoding());
5445 if (imm8 == 1) {
5446 emit_int16((unsigned char)0xD1, (0xC8 | encode));
5447 } else {
5448 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
5449 }
5450 }
5451
5452 void Assembler::rolq(Register dst) {
5453 int encode = prefixq_and_encode(dst->encoding());
5454 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5455 }
5456
5457 void Assembler::rolq(Register dst, int imm8) {
5458 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5459 int encode = prefixq_and_encode(dst->encoding());
5460 if (imm8 == 1) {
5461 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5462 } else {
5463 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
5464 }
5465 }
5466 #endif
5467
5468 void Assembler::sahf() {
5469 #ifdef _LP64
5470 // Not supported in 64bit mode
5471 ShouldNotReachHere();
5472 #endif
5473 emit_int8((unsigned char)0x9E);
5474 }
5475
5476 void Assembler::sall(Address dst, int imm8) {
5477 InstructionMark im(this);
5478 assert(isShiftCount(imm8), "illegal shift count");
5479 prefix(dst);
5480 if (imm8 == 1) {
5481 emit_int8((unsigned char)0xD1);
5482 emit_operand(as_Register(4), dst);
5483 }
5484 else {
5485 emit_int8((unsigned char)0xC1);
5486 emit_operand(as_Register(4), dst);
5487 emit_int8(imm8);
5488 }
5489 }
5490
5491 void Assembler::sall(Address dst) {
5492 InstructionMark im(this);
5493 prefix(dst);
5494 emit_int8((unsigned char)0xD3);
5495 emit_operand(as_Register(4), dst);
5496 }
5497
5498 void Assembler::sall(Register dst, int imm8) {
5499 assert(isShiftCount(imm8), "illegal shift count");
5500 int encode = prefix_and_encode(dst->encoding());
5501 if (imm8 == 1) {
5502 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5503 } else {
5504 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5505 }
5506 }
5507
5508 void Assembler::sall(Register dst) {
5509 int encode = prefix_and_encode(dst->encoding());
5510 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5511 }
5512
5513 void Assembler::sarl(Address dst, int imm8) {
5514 assert(isShiftCount(imm8), "illegal shift count");
5515 InstructionMark im(this);
5516 prefix(dst);
5517 if (imm8 == 1) {
5518 emit_int8((unsigned char)0xD1);
5519 emit_operand(as_Register(7), dst);
5520 }
5521 else {
5522 emit_int8((unsigned char)0xC1);
5523 emit_operand(as_Register(7), dst);
5524 emit_int8(imm8);
5525 }
5526 }
5527
5528 void Assembler::sarl(Address dst) {
5529 InstructionMark im(this);
5530 prefix(dst);
5531 emit_int8((unsigned char)0xD3);
5532 emit_operand(as_Register(7), dst);
5533 }
5534
5535 void Assembler::sarl(Register dst, int imm8) {
5536 int encode = prefix_and_encode(dst->encoding());
5537 assert(isShiftCount(imm8), "illegal shift count");
5538 if (imm8 == 1) {
5539 emit_int16((unsigned char)0xD1, (0xF8 | encode));
5540 } else {
5541 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
5542 }
5543 }
5544
5545 void Assembler::sarl(Register dst) {
5546 int encode = prefix_and_encode(dst->encoding());
5547 emit_int16((unsigned char)0xD3, (0xF8 | encode));
5548 }
5549
5550 void Assembler::sbbl(Address dst, int32_t imm32) {
5551 InstructionMark im(this);
5552 prefix(dst);
5553 emit_arith_operand(0x81, rbx, dst, imm32);
5554 }
5555
5556 void Assembler::sbbl(Register dst, int32_t imm32) {
5557 prefix(dst);
5558 emit_arith(0x81, 0xD8, dst, imm32);
5559 }
5560
5561
5562 void Assembler::sbbl(Register dst, Address src) {
5563 InstructionMark im(this);
5564 prefix(src, dst);
5565 emit_int8(0x1B);
5566 emit_operand(dst, src);
5567 }
5568
5569 void Assembler::sbbl(Register dst, Register src) {
5570 (void) prefix_and_encode(dst->encoding(), src->encoding());
5571 emit_arith(0x1B, 0xC0, dst, src);
5572 }
5573
5574 void Assembler::setb(Condition cc, Register dst) {
5575 assert(0 <= cc && cc < 16, "illegal cc");
5576 int encode = prefix_and_encode(dst->encoding(), true);
5577 emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
5578 }
5579
5580 void Assembler::sete(Register dst) {
5581 int encode = prefix_and_encode(dst->encoding(), true);
5582 emit_int24(0x0F, (unsigned char)0x94, (0xC0 | encode));
5583 }
5584
5585 void Assembler::setl(Register dst) {
5586 int encode = prefix_and_encode(dst->encoding(), true);
5587 emit_int24(0x0F, (unsigned char)0x9C, (0xC0 | encode));
5588 }
5589
5590 void Assembler::setne(Register dst) {
5591 int encode = prefix_and_encode(dst->encoding(), true);
5592 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | encode));
5593 }
5594
5595 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
5596 assert(VM_Version::supports_ssse3(), "");
5597 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5598 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5599 emit_int24(0x0F, (0xC0 | encode), imm8);
5600 }
5601
5602 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
5603 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
5604 vector_len == AVX_256bit? VM_Version::supports_avx2() :
5605 0, "");
5606 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5607 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5608 emit_int24(0x0F, (0xC0 | encode), imm8);
5609 }
5610
5611 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5612 assert(VM_Version::supports_evex(), "");
5613 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5614 attributes.set_is_evex_instruction();
5615 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5616 emit_int24(0x3, (0xC0 | encode), imm8);
5617 }
5618
5619 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
5620 assert(VM_Version::supports_sse4_1(), "");
5621 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5622 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5623 emit_int24(0x0E, (0xC0 | encode), imm8);
5624 }
5625
5626 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
5627 assert(VM_Version::supports_sha(), "");
5628 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
5629 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
5630 }
5631
5632 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
5633 assert(VM_Version::supports_sha(), "");
5634 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5635 emit_int16((unsigned char)0xC8, (0xC0 | encode));
5636 }
5637
5638 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
5639 assert(VM_Version::supports_sha(), "");
5640 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5641 emit_int16((unsigned char)0xC9, (0xC0 | encode));
5642 }
5643
5644 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
5645 assert(VM_Version::supports_sha(), "");
5646 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5647 emit_int16((unsigned char)0xCA, (0xC0 | encode));
5648 }
5649
5650 // xmm0 is implicit additional source to this instruction.
5651 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
5652 assert(VM_Version::supports_sha(), "");
5653 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5654 emit_int16((unsigned char)0xCB, (0xC0 | encode));
5655 }
5656
5657 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
5658 assert(VM_Version::supports_sha(), "");
5659 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5660 emit_int16((unsigned char)0xCC, (0xC0 | encode));
5661 }
5662
5663 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
5664 assert(VM_Version::supports_sha(), "");
5665 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
5666 emit_int16((unsigned char)0xCD, (0xC0 | encode));
5667 }
5668
5669
5670 void Assembler::shll(Register dst, int imm8) {
5671 assert(isShiftCount(imm8), "illegal shift count");
5672 int encode = prefix_and_encode(dst->encoding());
5673 if (imm8 == 1 ) {
5674 emit_int16((unsigned char)0xD1, (0xE0 | encode));
5675 } else {
5676 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
5677 }
5678 }
5679
5680 void Assembler::shll(Register dst) {
5681 int encode = prefix_and_encode(dst->encoding());
5682 emit_int16((unsigned char)0xD3, (0xE0 | encode));
5683 }
5684
5685 void Assembler::shrl(Register dst, int imm8) {
5686 assert(isShiftCount(imm8), "illegal shift count");
5687 int encode = prefix_and_encode(dst->encoding());
5688 if (imm8 == 1) {
5689 emit_int16((unsigned char)0xD1, (0xE8 | encode));
5690 }
5691 else {
5692 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
5693 }
5694 }
5695
5696 void Assembler::shrl(Register dst) {
5697 int encode = prefix_and_encode(dst->encoding());
5698 emit_int16((unsigned char)0xD3, (0xE8 | encode));
5699 }
5700
5701 void Assembler::shrl(Address dst) {
5702 InstructionMark im(this);
5703 prefix(dst);
5704 emit_int8((unsigned char)0xD3);
5705 emit_operand(as_Register(5), dst);
5706 }
5707
5708 void Assembler::shrl(Address dst, int imm8) {
5709 InstructionMark im(this);
5710 assert(isShiftCount(imm8), "illegal shift count");
5711 prefix(dst);
5712 if (imm8 == 1) {
5713 emit_int8((unsigned char)0xD1);
5714 emit_operand(as_Register(5), dst);
5715 }
5716 else {
5717 emit_int8((unsigned char)0xC1);
5718 emit_operand(as_Register(5), dst);
5719 emit_int8(imm8);
5720 }
5721 }
5722
5723
5724 void Assembler::shldl(Register dst, Register src) {
5725 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5726 emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
5727 }
5728
5729 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
5730 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5731 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
5732 }
5733
5734 void Assembler::shrdl(Register dst, Register src) {
5735 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5736 emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
5737 }
5738
5739 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
5740 int encode = prefix_and_encode(src->encoding(), dst->encoding());
5741 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
5742 }
5743
5744 // copies a single word from [esi] to [edi]
5745 void Assembler::smovl() {
5746 emit_int8((unsigned char)0xA5);
5747 }
5748
5749 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
5750 assert(VM_Version::supports_sse4_1(), "");
5751 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5752 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5753 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
5754 }
5755
5756 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
5757 assert(VM_Version::supports_sse4_1(), "");
5758 InstructionMark im(this);
5759 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5760 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5761 emit_int8(0x0B);
5762 emit_operand(dst, src);
5763 emit_int8((unsigned char)rmode);
5764 }
5765
5766 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
5767 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5768 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5769 attributes.set_rex_vex_w_reverted();
5770 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5771 emit_int16(0x51, (0xC0 | encode));
5772 }
5773
5774 void Assembler::sqrtsd(XMMRegister dst, Address src) {
5775 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5776 InstructionMark im(this);
5777 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5778 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5779 attributes.set_rex_vex_w_reverted();
5780 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5781 emit_int8(0x51);
5782 emit_operand(dst, src);
5783 }
5784
5785 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
5786 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5787 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5788 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5789 emit_int16(0x51, (0xC0 | encode));
5790 }
5791
5792 void Assembler::std() {
5793 emit_int8((unsigned char)0xFD);
5794 }
5795
5796 void Assembler::sqrtss(XMMRegister dst, Address src) {
5797 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5798 InstructionMark im(this);
5799 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5800 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5801 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5802 emit_int8(0x51);
5803 emit_operand(dst, src);
5804 }
5805
5806 void Assembler::stmxcsr( Address dst) {
5807 if (UseAVX > 0 ) {
5808 assert(VM_Version::supports_avx(), "");
5809 InstructionMark im(this);
5810 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5811 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5812 emit_int8((unsigned char)0xAE);
5813 emit_operand(as_Register(3), dst);
5814 } else {
5815 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5816 InstructionMark im(this);
5817 prefix(dst);
5818 emit_int16(0x0F, (unsigned char)0xAE);
5819 emit_operand(as_Register(3), dst);
5820 }
5821 }
5822
5823 void Assembler::subl(Address dst, int32_t imm32) {
5824 InstructionMark im(this);
5825 prefix(dst);
5826 emit_arith_operand(0x81, rbp, dst, imm32);
5827 }
5828
5829 void Assembler::subl(Address dst, Register src) {
5830 InstructionMark im(this);
5831 prefix(dst, src);
5832 emit_int8(0x29);
5833 emit_operand(src, dst);
5834 }
5835
5836 void Assembler::subl(Register dst, int32_t imm32) {
5837 prefix(dst);
5838 emit_arith(0x81, 0xE8, dst, imm32);
5839 }
5840
5841 // Force generation of a 4 byte immediate value even if it fits into 8bit
5842 void Assembler::subl_imm32(Register dst, int32_t imm32) {
5843 prefix(dst);
5844 emit_arith_imm32(0x81, 0xE8, dst, imm32);
5845 }
5846
5847 void Assembler::subl(Register dst, Address src) {
5848 InstructionMark im(this);
5849 prefix(src, dst);
5850 emit_int8(0x2B);
5851 emit_operand(dst, src);
5852 }
5853
5854 void Assembler::subl(Register dst, Register src) {
5855 (void) prefix_and_encode(dst->encoding(), src->encoding());
5856 emit_arith(0x2B, 0xC0, dst, src);
5857 }
5858
5859 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
5860 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5861 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5862 attributes.set_rex_vex_w_reverted();
5863 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5864 emit_int16(0x5C, (0xC0 | encode));
5865 }
5866
5867 void Assembler::subsd(XMMRegister dst, Address src) {
5868 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5869 InstructionMark im(this);
5870 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5871 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5872 attributes.set_rex_vex_w_reverted();
5873 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5874 emit_int8(0x5C);
5875 emit_operand(dst, src);
5876 }
5877
5878 void Assembler::subss(XMMRegister dst, XMMRegister src) {
5879 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5880 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
5881 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5882 emit_int16(0x5C, (0xC0 | encode));
5883 }
5884
5885 void Assembler::subss(XMMRegister dst, Address src) {
5886 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5887 InstructionMark im(this);
5888 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5889 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5890 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5891 emit_int8(0x5C);
5892 emit_operand(dst, src);
5893 }
5894
5895 void Assembler::testb(Register dst, int imm8) {
5896 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
5897 if (dst == rax) {
5898 emit_int8((unsigned char)0xA8);
5899 emit_int8(imm8);
5900 } else {
5901 (void) prefix_and_encode(dst->encoding(), true);
5902 emit_arith_b(0xF6, 0xC0, dst, imm8);
5903 }
5904 }
5905
5906 void Assembler::testb(Address dst, int imm8) {
5907 InstructionMark im(this);
5908 prefix(dst);
5909 emit_int8((unsigned char)0xF6);
5910 emit_operand(rax, dst, 1);
5911 emit_int8(imm8);
5912 }
5913
5914 void Assembler::testl(Address dst, int32_t imm32) {
5915 if (imm32 >= 0 && is8bit(imm32)) {
5916 testb(dst, imm32);
5917 return;
5918 }
5919 InstructionMark im(this);
5920 emit_int8((unsigned char)0xF7);
5921 emit_operand(as_Register(0), dst);
5922 emit_int32(imm32);
5923 }
5924
5925 void Assembler::testl(Register dst, int32_t imm32) {
5926 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
5927 testb(dst, imm32);
5928 return;
5929 }
5930 // not using emit_arith because test
5931 // doesn't support sign-extension of
5932 // 8bit operands
5933 if (dst == rax) {
5934 emit_int8((unsigned char)0xA9);
5935 emit_int32(imm32);
5936 } else {
5937 int encode = dst->encoding();
5938 encode = prefix_and_encode(encode);
5939 emit_int16((unsigned char)0xF7, (0xC0 | encode));
5940 emit_int32(imm32);
5941 }
5942 }
5943
5944 void Assembler::testl(Register dst, Register src) {
5945 (void) prefix_and_encode(dst->encoding(), src->encoding());
5946 emit_arith(0x85, 0xC0, dst, src);
5947 }
5948
5949 void Assembler::testl(Register dst, Address src) {
5950 InstructionMark im(this);
5951 prefix(src, dst);
5952 emit_int8((unsigned char)0x85);
5953 emit_operand(dst, src);
5954 }
5955
5956 void Assembler::tzcntl(Register dst, Register src) {
5957 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
5958 emit_int8((unsigned char)0xF3);
5959 int encode = prefix_and_encode(dst->encoding(), src->encoding());
5960 emit_int24(0x0F,
5961 (unsigned char)0xBC,
5962 0xC0 | encode);
5963 }
5964
5965 void Assembler::tzcntl(Register dst, Address src) {
5966 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
5967 InstructionMark im(this);
5968 emit_int8((unsigned char)0xF3);
5969 prefix(src, dst);
5970 emit_int16(0x0F, (unsigned char)0xBC);
5971 emit_operand(dst, src);
5972 }
5973
5974 void Assembler::tzcntq(Register dst, Register src) {
5975 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
5976 emit_int8((unsigned char)0xF3);
5977 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5978 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
5979 }
5980
5981 void Assembler::tzcntq(Register dst, Address src) {
5982 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
5983 InstructionMark im(this);
5984 emit_int8((unsigned char)0xF3);
5985 prefixq(src, dst);
5986 emit_int16(0x0F, (unsigned char)0xBC);
5987 emit_operand(dst, src);
5988 }
5989
5990 void Assembler::ucomisd(XMMRegister dst, Address src) {
5991 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5992 InstructionMark im(this);
5993 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5994 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5995 attributes.set_rex_vex_w_reverted();
5996 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5997 emit_int8(0x2E);
5998 emit_operand(dst, src);
5999 }
6000
6001 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
6002 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6003 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6004 attributes.set_rex_vex_w_reverted();
6005 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6006 emit_int16(0x2E, (0xC0 | encode));
6007 }
6008
6009 void Assembler::ucomiss(XMMRegister dst, Address src) {
6010 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6011 InstructionMark im(this);
6012 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6013 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6014 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6015 emit_int8(0x2E);
6016 emit_operand(dst, src);
6017 }
6018
6019 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
6020 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6021 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6022 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6023 emit_int16(0x2E, (0xC0 | encode));
6024 }
6025
6026 void Assembler::xabort(int8_t imm8) {
6027 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
6028 }
6029
6030 void Assembler::xaddb(Address dst, Register src) {
6031 InstructionMark im(this);
6032 prefix(dst, src, true);
6033 emit_int16(0x0F, (unsigned char)0xC0);
6034 emit_operand(src, dst);
6035 }
6036
6037 void Assembler::xaddw(Address dst, Register src) {
6038 InstructionMark im(this);
6039 emit_int8(0x66);
6040 prefix(dst, src);
6041 emit_int16(0x0F, (unsigned char)0xC1);
6042 emit_operand(src, dst);
6043 }
6044
6045 void Assembler::xaddl(Address dst, Register src) {
6046 InstructionMark im(this);
6047 prefix(dst, src);
6048 emit_int16(0x0F, (unsigned char)0xC1);
6049 emit_operand(src, dst);
6050 }
6051
6052 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
6053 InstructionMark im(this);
6054 relocate(rtype);
6055 if (abort.is_bound()) {
6056 address entry = target(abort);
6057 assert(entry != NULL, "abort entry NULL");
6058 intptr_t offset = entry - pc();
6059 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6060 emit_int32(offset - 6); // 2 opcode + 4 address
6061 } else {
6062 abort.add_patch_at(code(), locator());
6063 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
6064 emit_int32(0);
6065 }
6066 }
6067
6068 void Assembler::xchgb(Register dst, Address src) { // xchg
6069 InstructionMark im(this);
6070 prefix(src, dst, true);
6071 emit_int8((unsigned char)0x86);
6072 emit_operand(dst, src);
6073 }
6074
6075 void Assembler::xchgw(Register dst, Address src) { // xchg
6076 InstructionMark im(this);
6077 emit_int8(0x66);
6078 prefix(src, dst);
6079 emit_int8((unsigned char)0x87);
6080 emit_operand(dst, src);
6081 }
6082
6083 void Assembler::xchgl(Register dst, Address src) { // xchg
6084 InstructionMark im(this);
6085 prefix(src, dst);
6086 emit_int8((unsigned char)0x87);
6087 emit_operand(dst, src);
6088 }
6089
6090 void Assembler::xchgl(Register dst, Register src) {
6091 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6092 emit_int16((unsigned char)0x87, (0xC0 | encode));
6093 }
6094
6095 void Assembler::xend() {
6096 emit_int24(0x0F, 0x01, (unsigned char)0xD5);
6097 }
6098
6099 void Assembler::xgetbv() {
6100 emit_int24(0x0F, 0x01, (unsigned char)0xD0);
6101 }
6102
6103 void Assembler::xorl(Address dst, int32_t imm32) {
6104 InstructionMark im(this);
6105 prefix(dst);
6106 emit_arith_operand(0x81, as_Register(6), dst, imm32);
6107 }
6108
6109 void Assembler::xorl(Register dst, int32_t imm32) {
6110 prefix(dst);
6111 emit_arith(0x81, 0xF0, dst, imm32);
6112 }
6113
6114 void Assembler::xorl(Register dst, Address src) {
6115 InstructionMark im(this);
6116 prefix(src, dst);
6117 emit_int8(0x33);
6118 emit_operand(dst, src);
6119 }
6120
6121 void Assembler::xorl(Register dst, Register src) {
6122 (void) prefix_and_encode(dst->encoding(), src->encoding());
6123 emit_arith(0x33, 0xC0, dst, src);
6124 }
6125
6126 void Assembler::xorl(Address dst, Register src) {
6127 InstructionMark im(this);
6128 prefix(dst, src);
6129 emit_int8(0x31);
6130 emit_operand(src, dst);
6131 }
6132
6133 void Assembler::xorb(Register dst, Address src) {
6134 InstructionMark im(this);
6135 prefix(src, dst);
6136 emit_int8(0x32);
6137 emit_operand(dst, src);
6138 }
6139
6140 void Assembler::xorb(Address dst, Register src) {
6141 InstructionMark im(this);
6142 prefix(dst, src, true);
6143 emit_int8(0x30);
6144 emit_operand(src, dst);
6145 }
6146
6147 void Assembler::xorw(Register dst, Register src) {
6148 (void)prefix_and_encode(dst->encoding(), src->encoding());
6149 emit_arith(0x33, 0xC0, dst, src);
6150 }
6151
6152 // AVX 3-operands scalar float-point arithmetic instructions
6153
6154 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
6155 assert(VM_Version::supports_avx(), "");
6156 InstructionMark im(this);
6157 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6158 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6159 attributes.set_rex_vex_w_reverted();
6160 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6161 emit_int8(0x58);
6162 emit_operand(dst, src);
6163 }
6164
6165 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6166 assert(VM_Version::supports_avx(), "");
6167 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6168 attributes.set_rex_vex_w_reverted();
6169 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6170 emit_int16(0x58, (0xC0 | encode));
6171 }
6172
6173 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
6174 assert(VM_Version::supports_avx(), "");
6175 InstructionMark im(this);
6176 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6177 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6178 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6179 emit_int8(0x58);
6180 emit_operand(dst, src);
6181 }
6182
6183 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6184 assert(VM_Version::supports_avx(), "");
6185 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6186 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6187 emit_int16(0x58, (0xC0 | encode));
6188 }
6189
6190 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
6191 assert(VM_Version::supports_avx(), "");
6192 InstructionMark im(this);
6193 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6194 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6195 attributes.set_rex_vex_w_reverted();
6196 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6197 emit_int8(0x5E);
6198 emit_operand(dst, src);
6199 }
6200
6201 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6202 assert(VM_Version::supports_avx(), "");
6203 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6204 attributes.set_rex_vex_w_reverted();
6205 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6206 emit_int16(0x5E, (0xC0 | encode));
6207 }
6208
6209 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
6210 assert(VM_Version::supports_avx(), "");
6211 InstructionMark im(this);
6212 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6213 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6214 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6215 emit_int8(0x5E);
6216 emit_operand(dst, src);
6217 }
6218
6219 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6220 assert(VM_Version::supports_avx(), "");
6221 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6222 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6223 emit_int16(0x5E, (0xC0 | encode));
6224 }
6225
6226 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6227 assert(VM_Version::supports_fma(), "");
6228 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6229 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6230 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6231 }
6232
6233 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6234 assert(VM_Version::supports_fma(), "");
6235 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6236 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6237 emit_int16((unsigned char)0xB9, (0xC0 | encode));
6238 }
6239
6240 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
6241 assert(VM_Version::supports_avx(), "");
6242 InstructionMark im(this);
6243 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6244 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6245 attributes.set_rex_vex_w_reverted();
6246 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6247 emit_int8(0x59);
6248 emit_operand(dst, src);
6249 }
6250
6251 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6252 assert(VM_Version::supports_avx(), "");
6253 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6254 attributes.set_rex_vex_w_reverted();
6255 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6256 emit_int16(0x59, (0xC0 | encode));
6257 }
6258
6259 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
6260 assert(VM_Version::supports_avx(), "");
6261 InstructionMark im(this);
6262 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6263 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6264 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6265 emit_int8(0x59);
6266 emit_operand(dst, src);
6267 }
6268
6269 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6270 assert(VM_Version::supports_avx(), "");
6271 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6272 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6273 emit_int16(0x59, (0xC0 | encode));
6274 }
6275
6276 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
6277 assert(VM_Version::supports_avx(), "");
6278 InstructionMark im(this);
6279 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6280 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6281 attributes.set_rex_vex_w_reverted();
6282 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6283 emit_int8(0x5C);
6284 emit_operand(dst, src);
6285 }
6286
6287 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6288 assert(VM_Version::supports_avx(), "");
6289 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6290 attributes.set_rex_vex_w_reverted();
6291 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
6292 emit_int16(0x5C, (0xC0 | encode));
6293 }
6294
6295 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
6296 assert(VM_Version::supports_avx(), "");
6297 InstructionMark im(this);
6298 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6299 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6300 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6301 emit_int8(0x5C);
6302 emit_operand(dst, src);
6303 }
6304
6305 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
6306 assert(VM_Version::supports_avx(), "");
6307 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
6308 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
6309 emit_int16(0x5C, (0xC0 | encode));
6310 }
6311
6312 //====================VECTOR ARITHMETIC=====================================
6313
6314 // Float-point vector arithmetic
6315
6316 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
6317 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6318 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6319 attributes.set_rex_vex_w_reverted();
6320 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6321 emit_int16(0x58, (0xC0 | encode));
6322 }
6323
6324 void Assembler::addpd(XMMRegister dst, Address src) {
6325 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6326 InstructionMark im(this);
6327 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6328 attributes.set_rex_vex_w_reverted();
6329 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6330 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6331 emit_int8(0x58);
6332 emit_operand(dst, src);
6333 }
6334
6335
6336 void Assembler::addps(XMMRegister dst, XMMRegister src) {
6337 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6338 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6339 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6340 emit_int16(0x58, (0xC0 | encode));
6341 }
6342
6343 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6344 assert(VM_Version::supports_avx(), "");
6345 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6346 attributes.set_rex_vex_w_reverted();
6347 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6348 emit_int16(0x58, (0xC0 | encode));
6349 }
6350
6351 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6352 assert(VM_Version::supports_avx(), "");
6353 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6354 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6355 emit_int16(0x58, (0xC0 | encode));
6356 }
6357
6358 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6359 assert(VM_Version::supports_avx(), "");
6360 InstructionMark im(this);
6361 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6362 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6363 attributes.set_rex_vex_w_reverted();
6364 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6365 emit_int8(0x58);
6366 emit_operand(dst, src);
6367 }
6368
6369 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6370 assert(VM_Version::supports_avx(), "");
6371 InstructionMark im(this);
6372 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6373 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6374 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6375 emit_int8(0x58);
6376 emit_operand(dst, src);
6377 }
6378
6379 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
6380 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6381 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6382 attributes.set_rex_vex_w_reverted();
6383 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6384 emit_int16(0x5C, (0xC0 | encode));
6385 }
6386
6387 void Assembler::subps(XMMRegister dst, XMMRegister src) {
6388 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6389 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6390 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6391 emit_int16(0x5C, (0xC0 | encode));
6392 }
6393
6394 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6395 assert(VM_Version::supports_avx(), "");
6396 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6397 attributes.set_rex_vex_w_reverted();
6398 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6399 emit_int16(0x5C, (0xC0 | encode));
6400 }
6401
6402 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6403 assert(VM_Version::supports_avx(), "");
6404 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6405 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6406 emit_int16(0x5C, (0xC0 | encode));
6407 }
6408
6409 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6410 assert(VM_Version::supports_avx(), "");
6411 InstructionMark im(this);
6412 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6413 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6414 attributes.set_rex_vex_w_reverted();
6415 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6416 emit_int8(0x5C);
6417 emit_operand(dst, src);
6418 }
6419
6420 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6421 assert(VM_Version::supports_avx(), "");
6422 InstructionMark im(this);
6423 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6424 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6425 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6426 emit_int8(0x5C);
6427 emit_operand(dst, src);
6428 }
6429
6430 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
6431 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6432 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6433 attributes.set_rex_vex_w_reverted();
6434 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6435 emit_int16(0x59, (0xC0 | encode));
6436 }
6437
6438 void Assembler::mulpd(XMMRegister dst, Address src) {
6439 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6440 InstructionMark im(this);
6441 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6442 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6443 attributes.set_rex_vex_w_reverted();
6444 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6445 emit_int8(0x59);
6446 emit_operand(dst, src);
6447 }
6448
6449 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
6450 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6451 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6452 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6453 emit_int16(0x59, (0xC0 | encode));
6454 }
6455
6456 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6457 assert(VM_Version::supports_avx(), "");
6458 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6459 attributes.set_rex_vex_w_reverted();
6460 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6461 emit_int16(0x59, (0xC0 | encode));
6462 }
6463
6464 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6465 assert(VM_Version::supports_avx(), "");
6466 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6467 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6468 emit_int16(0x59, (0xC0 | encode));
6469 }
6470
6471 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6472 assert(VM_Version::supports_avx(), "");
6473 InstructionMark im(this);
6474 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6475 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6476 attributes.set_rex_vex_w_reverted();
6477 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6478 emit_int8(0x59);
6479 emit_operand(dst, src);
6480 }
6481
6482 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6483 assert(VM_Version::supports_avx(), "");
6484 InstructionMark im(this);
6485 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6486 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6487 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6488 emit_int8(0x59);
6489 emit_operand(dst, src);
6490 }
6491
6492 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6493 assert(VM_Version::supports_fma(), "");
6494 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6495 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6496 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6497 }
6498
6499 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
6500 assert(VM_Version::supports_fma(), "");
6501 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6502 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6503 emit_int16((unsigned char)0xB8, (0xC0 | encode));
6504 }
6505
6506 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6507 assert(VM_Version::supports_fma(), "");
6508 InstructionMark im(this);
6509 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6510 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6511 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6512 emit_int8((unsigned char)0xB8);
6513 emit_operand(dst, src2);
6514 }
6515
6516 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
6517 assert(VM_Version::supports_fma(), "");
6518 InstructionMark im(this);
6519 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6520 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6521 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6522 emit_int8((unsigned char)0xB8);
6523 emit_operand(dst, src2);
6524 }
6525
6526 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
6527 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6528 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6529 attributes.set_rex_vex_w_reverted();
6530 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6531 emit_int16(0x5E, (0xC0 | encode));
6532 }
6533
6534 void Assembler::divps(XMMRegister dst, XMMRegister src) {
6535 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6536 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6537 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6538 emit_int16(0x5E, (0xC0 | encode));
6539 }
6540
6541 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6542 assert(VM_Version::supports_avx(), "");
6543 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6544 attributes.set_rex_vex_w_reverted();
6545 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6546 emit_int16(0x5E, (0xC0 | encode));
6547 }
6548
6549 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6550 assert(VM_Version::supports_avx(), "");
6551 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6552 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6553 emit_int16(0x5E, (0xC0 | encode));
6554 }
6555
6556 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6557 assert(VM_Version::supports_avx(), "");
6558 InstructionMark im(this);
6559 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6560 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6561 attributes.set_rex_vex_w_reverted();
6562 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6563 emit_int8(0x5E);
6564 emit_operand(dst, src);
6565 }
6566
6567 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6568 assert(VM_Version::supports_avx(), "");
6569 InstructionMark im(this);
6570 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6571 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6572 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6573 emit_int8(0x5E);
6574 emit_operand(dst, src);
6575 }
6576
6577 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6578 assert(VM_Version::supports_avx(), "");
6579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6580 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6581 emit_int24(0x09, (0xC0 | encode), (rmode));
6582 }
6583
6584 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6585 assert(VM_Version::supports_avx(), "");
6586 InstructionMark im(this);
6587 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6588 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6589 emit_int8(0x09);
6590 emit_operand(dst, src);
6591 emit_int8((rmode));
6592 }
6593
6594 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
6595 assert(VM_Version::supports_evex(), "requires EVEX support");
6596 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6597 attributes.set_is_evex_instruction();
6598 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6599 emit_int24(0x09, (0xC0 | encode), (rmode));
6600 }
6601
6602 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
6603 assert(VM_Version::supports_evex(), "requires EVEX support");
6604 assert(dst != xnoreg, "sanity");
6605 InstructionMark im(this);
6606 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6607 attributes.set_is_evex_instruction();
6608 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6609 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6610 emit_int8(0x09);
6611 emit_operand(dst, src);
6612 emit_int8((rmode));
6613 }
6614
6615 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
6616 assert(VM_Version::supports_avx(), "");
6617 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6618 attributes.set_rex_vex_w_reverted();
6619 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6620 emit_int16(0x51, (0xC0 | encode));
6621 }
6622
6623 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
6624 assert(VM_Version::supports_avx(), "");
6625 InstructionMark im(this);
6626 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6627 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6628 attributes.set_rex_vex_w_reverted();
6629 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6630 emit_int8(0x51);
6631 emit_operand(dst, src);
6632 }
6633
6634 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
6635 assert(VM_Version::supports_avx(), "");
6636 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6637 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6638 emit_int16(0x51, (0xC0 | encode));
6639 }
6640
6641 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
6642 assert(VM_Version::supports_avx(), "");
6643 InstructionMark im(this);
6644 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6645 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6646 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6647 emit_int8(0x51);
6648 emit_operand(dst, src);
6649 }
6650
6651 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
6652 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6653 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6654 attributes.set_rex_vex_w_reverted();
6655 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6656 emit_int16(0x54, (0xC0 | encode));
6657 }
6658
6659 void Assembler::andps(XMMRegister dst, XMMRegister src) {
6660 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6661 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6663 emit_int16(0x54, (0xC0 | encode));
6664 }
6665
6666 void Assembler::andps(XMMRegister dst, Address src) {
6667 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6668 InstructionMark im(this);
6669 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6670 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6671 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6672 emit_int8(0x54);
6673 emit_operand(dst, src);
6674 }
6675
6676 void Assembler::andpd(XMMRegister dst, Address src) {
6677 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6678 InstructionMark im(this);
6679 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6680 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6681 attributes.set_rex_vex_w_reverted();
6682 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6683 emit_int8(0x54);
6684 emit_operand(dst, src);
6685 }
6686
6687 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6688 assert(VM_Version::supports_avx(), "");
6689 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6690 attributes.set_rex_vex_w_reverted();
6691 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6692 emit_int16(0x54, (0xC0 | encode));
6693 }
6694
6695 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6696 assert(VM_Version::supports_avx(), "");
6697 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6698 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6699 emit_int16(0x54, (0xC0 | encode));
6700 }
6701
6702 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6703 assert(VM_Version::supports_avx(), "");
6704 InstructionMark im(this);
6705 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6706 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6707 attributes.set_rex_vex_w_reverted();
6708 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6709 emit_int8(0x54);
6710 emit_operand(dst, src);
6711 }
6712
6713 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6714 assert(VM_Version::supports_avx(), "");
6715 InstructionMark im(this);
6716 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6717 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6718 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6719 emit_int8(0x54);
6720 emit_operand(dst, src);
6721 }
6722
6723 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
6724 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6725 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6726 attributes.set_rex_vex_w_reverted();
6727 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6728 emit_int8(0x15);
6729 emit_int8((0xC0 | encode));
6730 }
6731
6732 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
6733 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6734 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6735 attributes.set_rex_vex_w_reverted();
6736 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6737 emit_int16(0x14, (0xC0 | encode));
6738 }
6739
6740 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
6741 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6742 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6743 attributes.set_rex_vex_w_reverted();
6744 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6745 emit_int16(0x57, (0xC0 | encode));
6746 }
6747
6748 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
6749 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6750 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6751 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6752 emit_int16(0x57, (0xC0 | encode));
6753 }
6754
6755 void Assembler::xorpd(XMMRegister dst, Address src) {
6756 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6757 InstructionMark im(this);
6758 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6759 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6760 attributes.set_rex_vex_w_reverted();
6761 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6762 emit_int8(0x57);
6763 emit_operand(dst, src);
6764 }
6765
6766 void Assembler::xorps(XMMRegister dst, Address src) {
6767 NOT_LP64(assert(VM_Version::supports_sse(), ""));
6768 InstructionMark im(this);
6769 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6770 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6771 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6772 emit_int8(0x57);
6773 emit_operand(dst, src);
6774 }
6775
6776 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6777 assert(VM_Version::supports_avx(), "");
6778 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6779 attributes.set_rex_vex_w_reverted();
6780 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6781 emit_int16(0x57, (0xC0 | encode));
6782 }
6783
6784 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6785 assert(VM_Version::supports_avx(), "");
6786 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6787 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6788 emit_int16(0x57, (0xC0 | encode));
6789 }
6790
6791 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6792 assert(VM_Version::supports_avx(), "");
6793 InstructionMark im(this);
6794 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6795 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6796 attributes.set_rex_vex_w_reverted();
6797 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6798 emit_int8(0x57);
6799 emit_operand(dst, src);
6800 }
6801
6802 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6803 assert(VM_Version::supports_avx(), "");
6804 InstructionMark im(this);
6805 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
6806 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6807 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6808 emit_int8(0x57);
6809 emit_operand(dst, src);
6810 }
6811
6812 // Integer vector arithmetic
6813 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6814 assert(VM_Version::supports_avx() && (vector_len == 0) ||
6815 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
6816 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6817 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6818 emit_int16(0x01, (0xC0 | encode));
6819 }
6820
6821 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6822 assert(VM_Version::supports_avx() && (vector_len == 0) ||
6823 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
6824 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6825 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6826 emit_int16(0x02, (0xC0 | encode));
6827 }
6828
6829 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
6830 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6831 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6832 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6833 emit_int16((unsigned char)0xFC, (0xC0 | encode));
6834 }
6835
6836 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
6837 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6838 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6839 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6840 emit_int16((unsigned char)0xFD, (0xC0 | encode));
6841 }
6842
6843 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
6844 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6845 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6846 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6847 emit_int16((unsigned char)0xFE, (0xC0 | encode));
6848 }
6849
6850 void Assembler::paddd(XMMRegister dst, Address src) {
6851 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6852 InstructionMark im(this);
6853 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6854 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6855 emit_int8((unsigned char)0xFE);
6856 emit_operand(dst, src);
6857 }
6858
6859 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
6860 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6861 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6862 attributes.set_rex_vex_w_reverted();
6863 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6864 emit_int16((unsigned char)0xD4, (0xC0 | encode));
6865 }
6866
6867 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
6868 assert(VM_Version::supports_sse3(), "");
6869 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6870 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6871 emit_int16(0x01, (0xC0 | encode));
6872 }
6873
6874 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
6875 assert(VM_Version::supports_sse3(), "");
6876 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6877 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6878 emit_int16(0x02, (0xC0 | encode));
6879 }
6880
6881 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6882 assert(UseAVX > 0, "requires some form of AVX");
6883 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6884 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6885 emit_int16((unsigned char)0xFC, (0xC0 | encode));
6886 }
6887
6888 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6889 assert(UseAVX > 0, "requires some form of AVX");
6890 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6891 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6892 emit_int16((unsigned char)0xFD, (0xC0 | encode));
6893 }
6894
6895 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6896 assert(UseAVX > 0, "requires some form of AVX");
6897 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6898 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6899 emit_int16((unsigned char)0xFE, (0xC0 | encode));
6900 }
6901
6902 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6903 assert(UseAVX > 0, "requires some form of AVX");
6904 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6905 attributes.set_rex_vex_w_reverted();
6906 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6907 emit_int16((unsigned char)0xD4, (0xC0 | encode));
6908 }
6909
6910 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6911 assert(UseAVX > 0, "requires some form of AVX");
6912 InstructionMark im(this);
6913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6914 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
6915 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6916 emit_int8((unsigned char)0xFC);
6917 emit_operand(dst, src);
6918 }
6919
6920 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6921 assert(UseAVX > 0, "requires some form of AVX");
6922 InstructionMark im(this);
6923 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6924 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
6925 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6926 emit_int8((unsigned char)0xFD);
6927 emit_operand(dst, src);
6928 }
6929
6930 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6931 assert(UseAVX > 0, "requires some form of AVX");
6932 InstructionMark im(this);
6933 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6934 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6935 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6936 emit_int8((unsigned char)0xFE);
6937 emit_operand(dst, src);
6938 }
6939
6940 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6941 assert(UseAVX > 0, "requires some form of AVX");
6942 InstructionMark im(this);
6943 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6944 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6945 attributes.set_rex_vex_w_reverted();
6946 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6947 emit_int8((unsigned char)0xD4);
6948 emit_operand(dst, src);
6949 }
6950
6951 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
6952 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6953 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6954 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6955 emit_int16((unsigned char)0xF8, (0xC0 | encode));
6956 }
6957
6958 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
6959 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6960 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6961 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6962 emit_int16((unsigned char)0xF9, (0xC0 | encode));
6963 }
6964
6965 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
6966 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6967 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6968 emit_int16((unsigned char)0xFA, (0xC0 | encode));
6969 }
6970
6971 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
6972 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6973 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6974 attributes.set_rex_vex_w_reverted();
6975 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6976 emit_int8((unsigned char)0xFB);
6977 emit_int8((0xC0 | encode));
6978 }
6979
6980 void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6981 assert(UseAVX > 0, "requires some form of AVX");
6982 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6983 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6984 emit_int16((unsigned char)0xD8, (0xC0 | encode));
6985 }
6986
6987 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6988 assert(UseAVX > 0, "requires some form of AVX");
6989 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6990 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6991 emit_int16((unsigned char)0xF8, (0xC0 | encode));
6992 }
6993
6994 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6995 assert(UseAVX > 0, "requires some form of AVX");
6996 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6997 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6998 emit_int16((unsigned char)0xF9, (0xC0 | encode));
6999 }
7000
7001 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7002 assert(UseAVX > 0, "requires some form of AVX");
7003 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7004 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7005 emit_int16((unsigned char)0xFA, (0xC0 | encode));
7006 }
7007
7008 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7009 assert(UseAVX > 0, "requires some form of AVX");
7010 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7011 attributes.set_rex_vex_w_reverted();
7012 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7013 emit_int16((unsigned char)0xFB, (0xC0 | encode));
7014 }
7015
7016 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7017 assert(UseAVX > 0, "requires some form of AVX");
7018 InstructionMark im(this);
7019 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7020 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7021 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7022 emit_int8((unsigned char)0xF8);
7023 emit_operand(dst, src);
7024 }
7025
7026 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7027 assert(UseAVX > 0, "requires some form of AVX");
7028 InstructionMark im(this);
7029 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7030 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7031 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7032 emit_int8((unsigned char)0xF9);
7033 emit_operand(dst, src);
7034 }
7035
7036 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7037 assert(UseAVX > 0, "requires some form of AVX");
7038 InstructionMark im(this);
7039 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7040 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7041 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7042 emit_int8((unsigned char)0xFA);
7043 emit_operand(dst, src);
7044 }
7045
7046 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7047 assert(UseAVX > 0, "requires some form of AVX");
7048 InstructionMark im(this);
7049 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7050 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7051 attributes.set_rex_vex_w_reverted();
7052 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7053 emit_int8((unsigned char)0xFB);
7054 emit_operand(dst, src);
7055 }
7056
7057 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
7058 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7059 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7060 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7061 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7062 }
7063
7064 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
7065 assert(VM_Version::supports_sse4_1(), "");
7066 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7067 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7068 emit_int16(0x40, (0xC0 | encode));
7069 }
7070
7071 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) {
7072 assert(VM_Version::supports_sse2(), "");
7073 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7074 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7075 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7076 }
7077
7078 void Assembler::vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7079 assert((vector_len == AVX_128bit && VM_Version::supports_avx()) ||
7080 (vector_len == AVX_256bit && VM_Version::supports_avx2()) ||
7081 (vector_len == AVX_512bit && VM_Version::supports_avx512bw()), "");
7082 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7083 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7084 emit_int16((unsigned char)0xE4, (0xC0 | encode));
7085 }
7086
7087 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7088 assert(UseAVX > 0, "requires some form of AVX");
7089 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7090 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7091 emit_int16((unsigned char)0xD5, (0xC0 | encode));
7092 }
7093
7094 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7095 assert(UseAVX > 0, "requires some form of AVX");
7096 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7097 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7098 emit_int16(0x40, (0xC0 | encode));
7099 }
7100
7101 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7102 assert(UseAVX > 2, "requires some form of EVEX");
7103 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7104 attributes.set_is_evex_instruction();
7105 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7106 emit_int16(0x40, (0xC0 | encode));
7107 }
7108
7109 void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7110 assert(UseAVX > 0, "requires some form of AVX");
7111 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7112 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7113 emit_int16((unsigned char)0xF4, (0xC0 | encode));
7114 }
7115
7116 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7117 assert(UseAVX > 0, "requires some form of AVX");
7118 InstructionMark im(this);
7119 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7120 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
7121 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7122 emit_int8((unsigned char)0xD5);
7123 emit_operand(dst, src);
7124 }
7125
7126 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7127 assert(UseAVX > 0, "requires some form of AVX");
7128 InstructionMark im(this);
7129 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7130 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7131 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7132 emit_int8(0x40);
7133 emit_operand(dst, src);
7134 }
7135
7136 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7137 assert(UseAVX > 2, "requires some form of EVEX");
7138 InstructionMark im(this);
7139 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
7140 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7141 attributes.set_is_evex_instruction();
7142 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7143 emit_int8(0x40);
7144 emit_operand(dst, src);
7145 }
7146
7147 // Min, max
7148 void Assembler::pminsb(XMMRegister dst, XMMRegister src) {
7149 assert(VM_Version::supports_sse4_1(), "");
7150 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7151 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7152 emit_int16(0x38, (0xC0 | encode));
7153 }
7154
7155 void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7156 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7157 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7158 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7159 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7160 emit_int16(0x38, (0xC0 | encode));
7161 }
7162
7163 void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
7164 assert(VM_Version::supports_sse2(), "");
7165 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7166 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7167 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7168 }
7169
7170 void Assembler::vpminsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7171 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7172 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7173 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7174 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7175 emit_int16((unsigned char)0xEA, (0xC0 | encode));
7176 }
7177
7178 void Assembler::pminsd(XMMRegister dst, XMMRegister src) {
7179 assert(VM_Version::supports_sse4_1(), "");
7180 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7181 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7182 emit_int16(0x39, (0xC0 | encode));
7183 }
7184
7185 void Assembler::vpminsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7186 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7187 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7188 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7189 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7190 emit_int16(0x39, (0xC0 | encode));
7191 }
7192
7193 void Assembler::vpminsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7194 assert(UseAVX > 2, "requires AVX512F");
7195 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7196 attributes.set_is_evex_instruction();
7197 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7198 emit_int16(0x39, (0xC0 | encode));
7199 }
7200
7201 void Assembler::minps(XMMRegister dst, XMMRegister src) {
7202 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7203 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7204 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7205 emit_int16(0x5D, (0xC0 | encode));
7206 }
7207 void Assembler::vminps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7208 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7209 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7210 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7211 emit_int16(0x5D, (0xC0 | encode));
7212 }
7213
7214 void Assembler::minpd(XMMRegister dst, XMMRegister src) {
7215 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7216 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7217 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7218 emit_int16(0x5D, (0xC0 | encode));
7219 }
7220 void Assembler::vminpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7221 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7222 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7223 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7224 emit_int16(0x5D, (0xC0 | encode));
7225 }
7226
7227 void Assembler::pmaxsb(XMMRegister dst, XMMRegister src) {
7228 assert(VM_Version::supports_sse4_1(), "");
7229 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7230 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7231 emit_int16(0x3C, (0xC0 | encode));
7232 }
7233
7234 void Assembler::vpmaxsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7235 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7236 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7237 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7238 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7239 emit_int16(0x3C, (0xC0 | encode));
7240 }
7241
7242 void Assembler::pmaxsw(XMMRegister dst, XMMRegister src) {
7243 assert(VM_Version::supports_sse2(), "");
7244 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7245 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7246 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7247 }
7248
7249 void Assembler::vpmaxsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7250 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7251 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
7252 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7253 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7254 emit_int16((unsigned char)0xEE, (0xC0 | encode));
7255 }
7256
7257 void Assembler::pmaxsd(XMMRegister dst, XMMRegister src) {
7258 assert(VM_Version::supports_sse4_1(), "");
7259 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7260 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7261 emit_int16(0x3D, (0xC0 | encode));
7262 }
7263
7264 void Assembler::vpmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7265 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7266 (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
7267 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7268 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7269 emit_int16(0x3D, (0xC0 | encode));
7270 }
7271
7272 void Assembler::vpmaxsq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7273 assert(UseAVX > 2, "requires AVX512F");
7274 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7275 attributes.set_is_evex_instruction();
7276 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7277 emit_int16(0x3D, (0xC0 | encode));
7278 }
7279
7280 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
7281 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7282 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7283 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7284 emit_int16(0x5F, (0xC0 | encode));
7285 }
7286
7287 void Assembler::vmaxps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7288 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7289 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7290 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7291 emit_int16(0x5F, (0xC0 | encode));
7292 }
7293
7294 void Assembler::maxpd(XMMRegister dst, XMMRegister src) {
7295 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7296 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7297 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7298 emit_int16(0x5F, (0xC0 | encode));
7299 }
7300
7301 void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7302 assert(vector_len >= AVX_512bit ? VM_Version::supports_evex() : VM_Version::supports_avx(), "");
7303 InstructionAttr attributes(vector_len, /* vex_w */true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7304 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7305 emit_int16(0x5F, (0xC0 | encode));
7306 }
7307
7308 // Shift packed integers left by specified number of bits.
7309 void Assembler::psllw(XMMRegister dst, int shift) {
7310 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7311 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7312 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7313 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7314 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7315 }
7316
7317 void Assembler::pslld(XMMRegister dst, int shift) {
7318 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7319 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7320 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7321 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7322 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7323 }
7324
7325 void Assembler::psllq(XMMRegister dst, int shift) {
7326 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7327 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7328 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7329 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7330 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7331 }
7332
7333 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
7334 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7335 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7336 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7337 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7338 }
7339
7340 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
7341 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7342 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7343 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7344 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7345 }
7346
7347 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
7348 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7349 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7350 attributes.set_rex_vex_w_reverted();
7351 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7352 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7353 }
7354
7355 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7356 assert(UseAVX > 0, "requires some form of AVX");
7357 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7358 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
7359 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7360 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7361 }
7362
7363 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7364 assert(UseAVX > 0, "requires some form of AVX");
7365 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7366 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7367 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
7368 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7369 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7370 }
7371
7372 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7373 assert(UseAVX > 0, "requires some form of AVX");
7374 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7375 attributes.set_rex_vex_w_reverted();
7376 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
7377 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7378 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7379 }
7380
7381 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7382 assert(UseAVX > 0, "requires some form of AVX");
7383 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7384 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7385 emit_int16((unsigned char)0xF1, (0xC0 | encode));
7386 }
7387
7388 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7389 assert(UseAVX > 0, "requires some form of AVX");
7390 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7391 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7392 emit_int16((unsigned char)0xF2, (0xC0 | encode));
7393 }
7394
7395 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7396 assert(UseAVX > 0, "requires some form of AVX");
7397 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7398 attributes.set_rex_vex_w_reverted();
7399 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7400 emit_int16((unsigned char)0xF3, (0xC0 | encode));
7401 }
7402
7403 // Shift packed integers logically right by specified number of bits.
7404 void Assembler::psrlw(XMMRegister dst, int shift) {
7405 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7406 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7407 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7408 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7409 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7410 }
7411
7412 void Assembler::psrld(XMMRegister dst, int shift) {
7413 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7414 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7415 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7416 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7417 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7418 }
7419
7420 void Assembler::psrlq(XMMRegister dst, int shift) {
7421 // Do not confuse it with psrldq SSE2 instruction which
7422 // shifts 128 bit value in xmm register by number of bytes.
7423 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7424 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7425 attributes.set_rex_vex_w_reverted();
7426 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7427 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7428 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7429 }
7430
7431 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
7432 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7433 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7434 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7435 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7436 }
7437
7438 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
7439 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7440 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7441 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7442 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7443 }
7444
7445 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
7446 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7447 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7448 attributes.set_rex_vex_w_reverted();
7449 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7450 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7451 }
7452
7453 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7454 assert(UseAVX > 0, "requires some form of AVX");
7455 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7456 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
7457 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7458 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7459 }
7460
7461 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7462 assert(UseAVX > 0, "requires some form of AVX");
7463 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7464 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
7465 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7466 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7467 }
7468
7469 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7470 assert(UseAVX > 0, "requires some form of AVX");
7471 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7472 attributes.set_rex_vex_w_reverted();
7473 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
7474 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7475 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
7476 }
7477
7478 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7479 assert(UseAVX > 0, "requires some form of AVX");
7480 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7481 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7482 emit_int16((unsigned char)0xD1, (0xC0 | encode));
7483 }
7484
7485 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7486 assert(UseAVX > 0, "requires some form of AVX");
7487 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7488 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7489 emit_int16((unsigned char)0xD2, (0xC0 | encode));
7490 }
7491
7492 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7493 assert(UseAVX > 0, "requires some form of AVX");
7494 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7495 attributes.set_rex_vex_w_reverted();
7496 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7497 emit_int16((unsigned char)0xD3, (0xC0 | encode));
7498 }
7499
7500 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7501 assert(VM_Version::supports_avx512bw(), "");
7502 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7503 attributes.set_is_evex_instruction();
7504 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7505 emit_int16(0x10, (0xC0 | encode));
7506 }
7507
7508 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7509 assert(VM_Version::supports_avx512bw(), "");
7510 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7511 attributes.set_is_evex_instruction();
7512 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7513 emit_int16(0x12, (0xC0 | encode));
7514 }
7515
7516 // Shift packed integers arithmetically right by specified number of bits.
7517 void Assembler::psraw(XMMRegister dst, int shift) {
7518 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7519 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7520 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7521 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7522 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7523 }
7524
7525 void Assembler::psrad(XMMRegister dst, int shift) {
7526 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7527 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7528 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
7529 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7530 emit_int8(0x72);
7531 emit_int8((0xC0 | encode));
7532 emit_int8(shift & 0xFF);
7533 }
7534
7535 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
7536 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7537 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7538 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7539 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7540 }
7541
7542 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
7543 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7544 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7545 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7546 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7547 }
7548
7549 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7550 assert(UseAVX > 0, "requires some form of AVX");
7551 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7552 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7553 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7554 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
7555 }
7556
7557 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7558 assert(UseAVX > 0, "requires some form of AVX");
7559 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7560 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
7561 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7562 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7563 }
7564
7565 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7566 assert(UseAVX > 0, "requires some form of AVX");
7567 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
7568 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7569 emit_int16((unsigned char)0xE1, (0xC0 | encode));
7570 }
7571
7572 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7573 assert(UseAVX > 0, "requires some form of AVX");
7574 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7575 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7576 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7577 }
7578
7579 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7580 assert(UseAVX > 2, "requires AVX512");
7581 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7582 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7583 attributes.set_is_evex_instruction();
7584 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7585 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
7586 }
7587
7588 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7589 assert(UseAVX > 2, "requires AVX512");
7590 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
7591 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7592 attributes.set_is_evex_instruction();
7593 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7594 emit_int16((unsigned char)0xE2, (0xC0 | encode));
7595 }
7596
7597 // logical operations packed integers
7598 void Assembler::pand(XMMRegister dst, XMMRegister src) {
7599 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7600 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7601 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7602 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7603 }
7604
7605 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7606 assert(UseAVX > 0, "requires some form of AVX");
7607 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7608 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7609 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7610 }
7611
7612 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7613 assert(UseAVX > 0, "requires some form of AVX");
7614 InstructionMark im(this);
7615 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7616 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7617 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7618 emit_int8((unsigned char)0xDB);
7619 emit_operand(dst, src);
7620 }
7621
7622 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7623 assert(VM_Version::supports_evex(), "");
7624 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7625 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7626 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7627 }
7628
7629 //Variable Shift packed integers logically left.
7630 void Assembler::vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7631 assert(UseAVX > 1, "requires AVX2");
7632 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7633 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7634 emit_int16(0x47, (0xC0 | encode));
7635 }
7636
7637 void Assembler::vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7638 assert(UseAVX > 1, "requires AVX2");
7639 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7640 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7641 emit_int16(0x47, (0xC0 | encode));
7642 }
7643
7644 //Variable Shift packed integers logically right.
7645 void Assembler::vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7646 assert(UseAVX > 1, "requires AVX2");
7647 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7648 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7649 emit_int16(0x45, (0xC0 | encode));
7650 }
7651
7652 void Assembler::vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7653 assert(UseAVX > 1, "requires AVX2");
7654 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7655 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7656 emit_int16(0x45, (0xC0 | encode));
7657 }
7658
7659 //Variable right Shift arithmetic packed integers .
7660 void Assembler::vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7661 assert(UseAVX > 1, "requires AVX2");
7662 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7663 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7664 emit_int16(0x46, (0xC0 | encode));
7665 }
7666
7667 void Assembler::evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7668 assert(VM_Version::supports_avx512bw(), "");
7669 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7670 attributes.set_is_evex_instruction();
7671 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7672 emit_int16(0x11, (0xC0 | encode));
7673 }
7674
7675 void Assembler::evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7676 assert(UseAVX > 2, "requires AVX512");
7677 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires AVX512VL");
7678 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7679 attributes.set_is_evex_instruction();
7680 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7681 emit_int16(0x46, (0xC0 | encode));
7682 }
7683
7684 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7685 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7686 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7687 attributes.set_is_evex_instruction();
7688 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7689 emit_int16(0x71, (0xC0 | encode));
7690 }
7691
7692 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
7693 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
7694 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7695 attributes.set_is_evex_instruction();
7696 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7697 emit_int16(0x73, (0xC0 | encode));
7698 }
7699
7700 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
7701 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7702 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7703 attributes.set_rex_vex_w_reverted();
7704 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7705 emit_int16((unsigned char)0xDF, (0xC0 | encode));
7706 }
7707
7708 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7709 assert(UseAVX > 0, "requires some form of AVX");
7710 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7711 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7712 emit_int16((unsigned char)0xDF, (0xC0 | encode));
7713 }
7714
7715 void Assembler::por(XMMRegister dst, XMMRegister src) {
7716 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7717 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7718 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7719 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7720 }
7721
7722 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7723 assert(UseAVX > 0, "requires some form of AVX");
7724 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7725 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7726 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7727 }
7728
7729 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7730 assert(UseAVX > 0, "requires some form of AVX");
7731 InstructionMark im(this);
7732 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7733 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7734 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7735 emit_int8((unsigned char)0xEB);
7736 emit_operand(dst, src);
7737 }
7738
7739 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7740 assert(VM_Version::supports_evex(), "");
7741 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7742 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7743 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7744 }
7745
7746
7747 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7748 assert(VM_Version::supports_evex(), "");
7749 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
7750 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7751 attributes.set_is_evex_instruction();
7752 attributes.set_embedded_opmask_register_specifier(mask);
7753 if (merge) {
7754 attributes.reset_is_clear_context();
7755 }
7756 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7757 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7758 }
7759
7760 void Assembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7761 assert(VM_Version::supports_evex(), "");
7762 // Encoding: EVEX.NDS.XXX.66.0F.W0 EB /r
7763 InstructionMark im(this);
7764 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7765 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
7766 attributes.set_is_evex_instruction();
7767 attributes.set_embedded_opmask_register_specifier(mask);
7768 if (merge) {
7769 attributes.reset_is_clear_context();
7770 }
7771 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7772 emit_int8((unsigned char)0xEB);
7773 emit_operand(dst, src);
7774 }
7775
7776 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
7777 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7778 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7779 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7780 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7781 }
7782
7783 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7784 assert(UseAVX > 0, "requires some form of AVX");
7785 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7786 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
7787 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
7788 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7789 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7790 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7791 }
7792
7793 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7794 assert(UseAVX > 0, "requires some form of AVX");
7795 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
7796 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
7797 vector_len == AVX_512bit ? VM_Version::supports_evex() : 0, "");
7798 InstructionMark im(this);
7799 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7800 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
7801 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7802 emit_int8((unsigned char)0xEF);
7803 emit_operand(dst, src);
7804 }
7805
7806 void Assembler::vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7807 assert(UseAVX > 2, "requires some form of EVEX");
7808 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7809 attributes.set_rex_vex_w_reverted();
7810 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7811 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7812 }
7813
7814 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7815 // Encoding: EVEX.NDS.XXX.66.0F.W0 EF /r
7816 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7817 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7818 attributes.set_is_evex_instruction();
7819 attributes.set_embedded_opmask_register_specifier(mask);
7820 if (merge) {
7821 attributes.reset_is_clear_context();
7822 }
7823 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7824 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7825 }
7826
7827 void Assembler::evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7828 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7829 InstructionMark im(this);
7830 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7831 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7832 attributes.set_is_evex_instruction();
7833 attributes.set_embedded_opmask_register_specifier(mask);
7834 if (merge) {
7835 attributes.reset_is_clear_context();
7836 }
7837 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7838 emit_int8((unsigned char)0xEF);
7839 emit_operand(dst, src);
7840 }
7841
7842 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7843 // Encoding: EVEX.NDS.XXX.66.0F.W1 EF /r
7844 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7845 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7846 attributes.set_is_evex_instruction();
7847 attributes.set_embedded_opmask_register_specifier(mask);
7848 if (merge) {
7849 attributes.reset_is_clear_context();
7850 }
7851 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7852 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7853 }
7854
7855 void Assembler::evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7856 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7857 InstructionMark im(this);
7858 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7859 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7860 attributes.set_is_evex_instruction();
7861 attributes.set_embedded_opmask_register_specifier(mask);
7862 if (merge) {
7863 attributes.reset_is_clear_context();
7864 }
7865 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7866 emit_int8((unsigned char)0xEF);
7867 emit_operand(dst, src);
7868 }
7869
7870 void Assembler::evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7871 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7872 InstructionMark im(this);
7873 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7874 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7875 attributes.set_is_evex_instruction();
7876 attributes.set_embedded_opmask_register_specifier(mask);
7877 if (merge) {
7878 attributes.reset_is_clear_context();
7879 }
7880 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7881 emit_int8((unsigned char)0xDB);
7882 emit_operand(dst, src);
7883 }
7884
7885 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7886 assert(VM_Version::supports_evex(), "");
7887 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7888 attributes.set_is_evex_instruction();
7889 attributes.set_embedded_opmask_register_specifier(mask);
7890 if (merge) {
7891 attributes.reset_is_clear_context();
7892 }
7893 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7894 emit_int16((unsigned char)0xDB, (0xC0 | encode));
7895 }
7896
7897 void Assembler::evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7898 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7899 InstructionMark im(this);
7900 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7901 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7902 attributes.set_is_evex_instruction();
7903 attributes.set_embedded_opmask_register_specifier(mask);
7904 if (merge) {
7905 attributes.reset_is_clear_context();
7906 }
7907 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7908 emit_int8((unsigned char)0xDB);
7909 emit_operand(dst, src);
7910 }
7911
7912 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
7913 assert(VM_Version::supports_evex(), "");
7914 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
7915 attributes.set_is_evex_instruction();
7916 attributes.set_embedded_opmask_register_specifier(mask);
7917 if (merge) {
7918 attributes.reset_is_clear_context();
7919 }
7920 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7921 emit_int16((unsigned char)0xEB, (0xC0 | encode));
7922 }
7923
7924 void Assembler::evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
7925 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
7926 InstructionMark im(this);
7927 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
7928 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
7929 attributes.set_is_evex_instruction();
7930 attributes.set_embedded_opmask_register_specifier(mask);
7931 if (merge) {
7932 attributes.reset_is_clear_context();
7933 }
7934 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7935 emit_int8((unsigned char)0xEB);
7936 emit_operand(dst, src);
7937 }
7938
7939 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
7940 assert(VM_Version::supports_evex(), "requires EVEX support");
7941 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7942 attributes.set_is_evex_instruction();
7943 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7944 emit_int16((unsigned char)0xEF, (0xC0 | encode));
7945 }
7946
7947 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
7948 assert(VM_Version::supports_evex(), "requires EVEX support");
7949 assert(dst != xnoreg, "sanity");
7950 InstructionMark im(this);
7951 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7952 attributes.set_is_evex_instruction();
7953 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
7954 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7955 emit_int8((unsigned char)0xEF);
7956 emit_operand(dst, src);
7957 }
7958
7959 void Assembler::evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7960 assert(VM_Version::supports_evex(), "requires EVEX support");
7961 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
7962 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7963 attributes.set_is_evex_instruction();
7964 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7965 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7966 }
7967
7968 void Assembler::evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7969 assert(VM_Version::supports_evex(), "requires EVEX support");
7970 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
7971 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7972 attributes.set_is_evex_instruction();
7973 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7974 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7975 }
7976
7977 // Register is a class, but it would be assigned numerical value.
7978 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
7979 PRAGMA_DIAG_PUSH
7980 PRAGMA_NONNULL_IGNORED
7981 void Assembler::evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7982 assert(VM_Version::supports_evex(), "requires EVEX support");
7983 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
7984 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7985 attributes.set_is_evex_instruction();
7986 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7987 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7988 }
7989
7990 void Assembler::evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
7991 assert(VM_Version::supports_evex(), "requires EVEX support");
7992 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
7993 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
7994 attributes.set_is_evex_instruction();
7995 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7996 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
7997 }
7998 PRAGMA_DIAG_POP
7999
8000 void Assembler::evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8001 assert(VM_Version::supports_evex(), "requires EVEX support");
8002 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8003 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8004 attributes.set_is_evex_instruction();
8005 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8006 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8007 }
8008
8009 void Assembler::evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8010 assert(VM_Version::supports_evex(), "requires EVEX support");
8011 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8012 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8013 attributes.set_is_evex_instruction();
8014 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8015 emit_int16(0x15, (unsigned char)(0xC0 | encode));
8016 }
8017
8018 void Assembler::evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8019 assert(VM_Version::supports_evex(), "requires EVEX support");
8020 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8021 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8022 attributes.set_is_evex_instruction();
8023 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8024 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8025 }
8026
8027 void Assembler::evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
8028 assert(VM_Version::supports_evex(), "requires EVEX support");
8029 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8030 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8031 attributes.set_is_evex_instruction();
8032 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8033 emit_int16(0x14, (unsigned char)(0xC0 | encode));
8034 }
8035
8036 void Assembler::evplzcntd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8037 assert(VM_Version::supports_avx512cd(), "");
8038 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8039 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8040 attributes.set_is_evex_instruction();
8041 attributes.set_embedded_opmask_register_specifier(mask);
8042 if (merge) {
8043 attributes.reset_is_clear_context();
8044 }
8045 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8046 emit_int16(0x44, (0xC0 | encode));
8047 }
8048
8049 void Assembler::evplzcntq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8050 assert(VM_Version::supports_avx512cd(), "");
8051 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8052 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8053 attributes.set_is_evex_instruction();
8054 attributes.set_embedded_opmask_register_specifier(mask);
8055 if (merge) {
8056 attributes.reset_is_clear_context();
8057 }
8058 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8059 emit_int16(0x44, (0xC0 | encode));
8060 }
8061
8062 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8063 assert(VM_Version::supports_evex(), "requires EVEX support");
8064 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8065 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8066 attributes.set_is_evex_instruction();
8067 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8068 emit_int8(0x25);
8069 emit_int8((unsigned char)(0xC0 | encode));
8070 emit_int8(imm8);
8071 }
8072
8073 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
8074 assert(VM_Version::supports_evex(), "requires EVEX support");
8075 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8076 assert(dst != xnoreg, "sanity");
8077 InstructionMark im(this);
8078 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8079 attributes.set_is_evex_instruction();
8080 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
8081 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8082 emit_int8(0x25);
8083 emit_operand(dst, src3);
8084 emit_int8(imm8);
8085 }
8086
8087 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
8088 assert(VM_Version::supports_evex(), "requires EVEX support");
8089 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
8090 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8091 attributes.set_is_evex_instruction();
8092 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8093 emit_int8(0x25);
8094 emit_int8((unsigned char)(0xC0 | encode));
8095 emit_int8(imm8);
8096 }
8097
8098 void Assembler::evexpandps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8099 assert(VM_Version::supports_evex(), "");
8100 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8101 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8102 attributes.set_is_evex_instruction();
8103 attributes.set_embedded_opmask_register_specifier(mask);
8104 if (merge) {
8105 attributes.reset_is_clear_context();
8106 }
8107 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8108 emit_int16((unsigned char)0x88, (0xC0 | encode));
8109 }
8110
8111 void Assembler::evexpandpd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8112 assert(VM_Version::supports_evex(), "");
8113 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8114 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8115 attributes.set_is_evex_instruction();
8116 attributes.set_embedded_opmask_register_specifier(mask);
8117 if (merge) {
8118 attributes.reset_is_clear_context();
8119 }
8120 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8121 emit_int16((unsigned char)0x88, (0xC0 | encode));
8122 }
8123
8124 void Assembler::evpexpandb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8125 assert(VM_Version::supports_avx512_vbmi2(), "");
8126 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8127 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8128 attributes.set_is_evex_instruction();
8129 attributes.set_embedded_opmask_register_specifier(mask);
8130 if (merge) {
8131 attributes.reset_is_clear_context();
8132 }
8133 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8134 emit_int16(0x62, (0xC0 | encode));
8135 }
8136
8137 void Assembler::evpexpandw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8138 assert(VM_Version::supports_avx512_vbmi2(), "");
8139 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8140 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8141 attributes.set_is_evex_instruction();
8142 attributes.set_embedded_opmask_register_specifier(mask);
8143 if (merge) {
8144 attributes.reset_is_clear_context();
8145 }
8146 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8147 emit_int16(0x62, (0xC0 | encode));
8148 }
8149
8150 void Assembler::evpexpandd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8151 assert(VM_Version::supports_evex(), "");
8152 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8153 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8154 attributes.set_is_evex_instruction();
8155 attributes.set_embedded_opmask_register_specifier(mask);
8156 if (merge) {
8157 attributes.reset_is_clear_context();
8158 }
8159 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8160 emit_int16((unsigned char)0x89, (0xC0 | encode));
8161 }
8162
8163 void Assembler::evpexpandq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
8164 assert(VM_Version::supports_evex(), "");
8165 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8166 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8167 attributes.set_is_evex_instruction();
8168 attributes.set_embedded_opmask_register_specifier(mask);
8169 if (merge) {
8170 attributes.reset_is_clear_context();
8171 }
8172 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8173 emit_int16((unsigned char)0x89, (0xC0 | encode));
8174 }
8175
8176 // vinserti forms
8177
8178 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8179 assert(VM_Version::supports_avx2(), "");
8180 assert(imm8 <= 0x01, "imm8: %u", imm8);
8181 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8182 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8183 // last byte:
8184 // 0x00 - insert into lower 128 bits
8185 // 0x01 - insert into upper 128 bits
8186 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
8187 }
8188
8189 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8190 assert(VM_Version::supports_avx2(), "");
8191 assert(dst != xnoreg, "sanity");
8192 assert(imm8 <= 0x01, "imm8: %u", imm8);
8193 InstructionMark im(this);
8194 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8195 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8196 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8197 emit_int8(0x38);
8198 emit_operand(dst, src);
8199 // 0x00 - insert into lower 128 bits
8200 // 0x01 - insert into upper 128 bits
8201 emit_int8(imm8 & 0x01);
8202 }
8203
8204 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8205 assert(VM_Version::supports_evex(), "");
8206 assert(imm8 <= 0x03, "imm8: %u", imm8);
8207 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8208 attributes.set_is_evex_instruction();
8209 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8210 // imm8:
8211 // 0x00 - insert into q0 128 bits (0..127)
8212 // 0x01 - insert into q1 128 bits (128..255)
8213 // 0x02 - insert into q2 128 bits (256..383)
8214 // 0x03 - insert into q3 128 bits (384..511)
8215 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
8216 }
8217
8218 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8219 assert(VM_Version::supports_evex(), "");
8220 assert(dst != xnoreg, "sanity");
8221 assert(imm8 <= 0x03, "imm8: %u", imm8);
8222 InstructionMark im(this);
8223 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8224 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8225 attributes.set_is_evex_instruction();
8226 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8227 emit_int8(0x18);
8228 emit_operand(dst, src);
8229 // 0x00 - insert into q0 128 bits (0..127)
8230 // 0x01 - insert into q1 128 bits (128..255)
8231 // 0x02 - insert into q2 128 bits (256..383)
8232 // 0x03 - insert into q3 128 bits (384..511)
8233 emit_int8(imm8 & 0x03);
8234 }
8235
8236 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8237 assert(VM_Version::supports_evex(), "");
8238 assert(imm8 <= 0x01, "imm8: %u", imm8);
8239 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8240 attributes.set_is_evex_instruction();
8241 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8242 //imm8:
8243 // 0x00 - insert into lower 256 bits
8244 // 0x01 - insert into upper 256 bits
8245 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
8246 }
8247
8248
8249 // vinsertf forms
8250
8251 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8252 assert(VM_Version::supports_avx(), "");
8253 assert(imm8 <= 0x01, "imm8: %u", imm8);
8254 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8255 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8256 // imm8:
8257 // 0x00 - insert into lower 128 bits
8258 // 0x01 - insert into upper 128 bits
8259 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
8260 }
8261
8262 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8263 assert(VM_Version::supports_avx(), "");
8264 assert(dst != xnoreg, "sanity");
8265 assert(imm8 <= 0x01, "imm8: %u", imm8);
8266 InstructionMark im(this);
8267 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8268 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8269 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8270 emit_int8(0x18);
8271 emit_operand(dst, src);
8272 // 0x00 - insert into lower 128 bits
8273 // 0x01 - insert into upper 128 bits
8274 emit_int8(imm8 & 0x01);
8275 }
8276
8277 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8278 assert(VM_Version::supports_evex(), "");
8279 assert(imm8 <= 0x03, "imm8: %u", imm8);
8280 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8282 // imm8:
8283 // 0x00 - insert into q0 128 bits (0..127)
8284 // 0x01 - insert into q1 128 bits (128..255)
8285 // 0x02 - insert into q0 128 bits (256..383)
8286 // 0x03 - insert into q1 128 bits (384..512)
8287 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
8288 }
8289
8290 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8291 assert(VM_Version::supports_evex(), "");
8292 assert(dst != xnoreg, "sanity");
8293 assert(imm8 <= 0x03, "imm8: %u", imm8);
8294 InstructionMark im(this);
8295 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8296 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8297 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8298 emit_int8(0x18);
8299 emit_operand(dst, src);
8300 // 0x00 - insert into q0 128 bits (0..127)
8301 // 0x01 - insert into q1 128 bits (128..255)
8302 // 0x02 - insert into q0 128 bits (256..383)
8303 // 0x03 - insert into q1 128 bits (384..512)
8304 emit_int8(imm8 & 0x03);
8305 }
8306
8307 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
8308 assert(VM_Version::supports_evex(), "");
8309 assert(imm8 <= 0x01, "imm8: %u", imm8);
8310 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8311 attributes.set_is_evex_instruction();
8312 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8313 // imm8:
8314 // 0x00 - insert into lower 256 bits
8315 // 0x01 - insert into upper 256 bits
8316 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
8317 }
8318
8319 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
8320 assert(VM_Version::supports_evex(), "");
8321 assert(dst != xnoreg, "sanity");
8322 assert(imm8 <= 0x01, "imm8: %u", imm8);
8323 InstructionMark im(this);
8324 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8325 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8326 attributes.set_is_evex_instruction();
8327 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8328 emit_int8(0x1A);
8329 emit_operand(dst, src);
8330 // 0x00 - insert into lower 256 bits
8331 // 0x01 - insert into upper 256 bits
8332 emit_int8(imm8 & 0x01);
8333 }
8334
8335
8336 // vextracti forms
8337
8338 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8339 assert(VM_Version::supports_avx2(), "");
8340 assert(imm8 <= 0x01, "imm8: %u", imm8);
8341 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8342 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8343 // imm8:
8344 // 0x00 - extract from lower 128 bits
8345 // 0x01 - extract from upper 128 bits
8346 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
8347 }
8348
8349 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
8350 assert(VM_Version::supports_avx2(), "");
8351 assert(src != xnoreg, "sanity");
8352 assert(imm8 <= 0x01, "imm8: %u", imm8);
8353 InstructionMark im(this);
8354 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8355 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8356 attributes.reset_is_clear_context();
8357 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8358 emit_int8(0x39);
8359 emit_operand(src, dst);
8360 // 0x00 - extract from lower 128 bits
8361 // 0x01 - extract from upper 128 bits
8362 emit_int8(imm8 & 0x01);
8363 }
8364
8365 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8366 assert(VM_Version::supports_evex(), "");
8367 assert(imm8 <= 0x03, "imm8: %u", imm8);
8368 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8369 attributes.set_is_evex_instruction();
8370 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8371 // imm8:
8372 // 0x00 - extract from bits 127:0
8373 // 0x01 - extract from bits 255:128
8374 // 0x02 - extract from bits 383:256
8375 // 0x03 - extract from bits 511:384
8376 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8377 }
8378
8379 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
8380 assert(VM_Version::supports_evex(), "");
8381 assert(src != xnoreg, "sanity");
8382 assert(imm8 <= 0x03, "imm8: %u", imm8);
8383 InstructionMark im(this);
8384 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8385 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8386 attributes.reset_is_clear_context();
8387 attributes.set_is_evex_instruction();
8388 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8389 emit_int8(0x39);
8390 emit_operand(src, dst);
8391 // 0x00 - extract from bits 127:0
8392 // 0x01 - extract from bits 255:128
8393 // 0x02 - extract from bits 383:256
8394 // 0x03 - extract from bits 511:384
8395 emit_int8(imm8 & 0x03);
8396 }
8397
8398 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8399 assert(VM_Version::supports_avx512dq(), "");
8400 assert(imm8 <= 0x03, "imm8: %u", imm8);
8401 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8402 attributes.set_is_evex_instruction();
8403 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8404 // imm8:
8405 // 0x00 - extract from bits 127:0
8406 // 0x01 - extract from bits 255:128
8407 // 0x02 - extract from bits 383:256
8408 // 0x03 - extract from bits 511:384
8409 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
8410 }
8411
8412 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8413 assert(VM_Version::supports_evex(), "");
8414 assert(imm8 <= 0x01, "imm8: %u", imm8);
8415 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8416 attributes.set_is_evex_instruction();
8417 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8418 // imm8:
8419 // 0x00 - extract from lower 256 bits
8420 // 0x01 - extract from upper 256 bits
8421 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
8422 }
8423
8424 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
8425 assert(VM_Version::supports_evex(), "");
8426 assert(src != xnoreg, "sanity");
8427 assert(imm8 <= 0x01, "imm8: %u", imm8);
8428 InstructionMark im(this);
8429 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8430 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
8431 attributes.reset_is_clear_context();
8432 attributes.set_is_evex_instruction();
8433 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8434 emit_int8(0x38);
8435 emit_operand(src, dst);
8436 // 0x00 - extract from lower 256 bits
8437 // 0x01 - extract from upper 256 bits
8438 emit_int8(imm8 & 0x01);
8439 }
8440 // vextractf forms
8441
8442 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8443 assert(VM_Version::supports_avx(), "");
8444 assert(imm8 <= 0x01, "imm8: %u", imm8);
8445 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8446 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8447 // imm8:
8448 // 0x00 - extract from lower 128 bits
8449 // 0x01 - extract from upper 128 bits
8450 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
8451 }
8452
8453 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
8454 assert(VM_Version::supports_avx(), "");
8455 assert(src != xnoreg, "sanity");
8456 assert(imm8 <= 0x01, "imm8: %u", imm8);
8457 InstructionMark im(this);
8458 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8459 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8460 attributes.reset_is_clear_context();
8461 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8462 emit_int8(0x19);
8463 emit_operand(src, dst);
8464 // 0x00 - extract from lower 128 bits
8465 // 0x01 - extract from upper 128 bits
8466 emit_int8(imm8 & 0x01);
8467 }
8468
8469 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8470 assert(VM_Version::supports_evex(), "");
8471 assert(imm8 <= 0x03, "imm8: %u", imm8);
8472 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8473 attributes.set_is_evex_instruction();
8474 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8475 // imm8:
8476 // 0x00 - extract from bits 127:0
8477 // 0x01 - extract from bits 255:128
8478 // 0x02 - extract from bits 383:256
8479 // 0x03 - extract from bits 511:384
8480 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8481 }
8482
8483 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
8484 assert(VM_Version::supports_evex(), "");
8485 assert(src != xnoreg, "sanity");
8486 assert(imm8 <= 0x03, "imm8: %u", imm8);
8487 InstructionMark im(this);
8488 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8489 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
8490 attributes.reset_is_clear_context();
8491 attributes.set_is_evex_instruction();
8492 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8493 emit_int8(0x19);
8494 emit_operand(src, dst);
8495 // 0x00 - extract from bits 127:0
8496 // 0x01 - extract from bits 255:128
8497 // 0x02 - extract from bits 383:256
8498 // 0x03 - extract from bits 511:384
8499 emit_int8(imm8 & 0x03);
8500 }
8501
8502 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8503 assert(VM_Version::supports_avx512dq(), "");
8504 assert(imm8 <= 0x03, "imm8: %u", imm8);
8505 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8506 attributes.set_is_evex_instruction();
8507 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8508 // imm8:
8509 // 0x00 - extract from bits 127:0
8510 // 0x01 - extract from bits 255:128
8511 // 0x02 - extract from bits 383:256
8512 // 0x03 - extract from bits 511:384
8513 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
8514 }
8515
8516 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
8517 assert(VM_Version::supports_evex(), "");
8518 assert(imm8 <= 0x01, "imm8: %u", imm8);
8519 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8520 attributes.set_is_evex_instruction();
8521 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8522 // imm8:
8523 // 0x00 - extract from lower 256 bits
8524 // 0x01 - extract from upper 256 bits
8525 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
8526 }
8527
8528 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
8529 assert(VM_Version::supports_evex(), "");
8530 assert(src != xnoreg, "sanity");
8531 assert(imm8 <= 0x01, "imm8: %u", imm8);
8532 InstructionMark im(this);
8533 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8534 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
8535 attributes.reset_is_clear_context();
8536 attributes.set_is_evex_instruction();
8537 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
8538 emit_int8(0x1B);
8539 emit_operand(src, dst);
8540 // 0x00 - extract from lower 256 bits
8541 // 0x01 - extract from upper 256 bits
8542 emit_int8(imm8 & 0x01);
8543 }
8544
8545 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8546 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
8547 assert(VM_Version::supports_avx2(), "");
8548 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8549 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8550 emit_int16(0x78, (0xC0 | encode));
8551 }
8552
8553 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
8554 assert(VM_Version::supports_avx2(), "");
8555 assert(dst != xnoreg, "sanity");
8556 InstructionMark im(this);
8557 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8558 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
8559 // swap src<->dst for encoding
8560 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8561 emit_int8(0x78);
8562 emit_operand(dst, src);
8563 }
8564
8565 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
8566 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
8567 assert(VM_Version::supports_avx2(), "");
8568 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8569 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8570 emit_int16(0x79, (0xC0 | encode));
8571 }
8572
8573 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
8574 assert(VM_Version::supports_avx2(), "");
8575 assert(dst != xnoreg, "sanity");
8576 InstructionMark im(this);
8577 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8578 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
8579 // swap src<->dst for encoding
8580 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8581 emit_int8(0x79);
8582 emit_operand(dst, src);
8583 }
8584
8585 void Assembler::vpsadbw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8586 assert(UseAVX > 0, "requires some form of AVX");
8587 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
8588 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8589 emit_int16((unsigned char)0xF6, (0xC0 | encode));
8590 }
8591
8592 void Assembler::vpunpckhwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8593 assert(UseAVX > 0, "requires some form of AVX");
8594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8596 emit_int16(0x69, (0xC0 | encode));
8597 }
8598
8599 void Assembler::vpunpcklwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8600 assert(UseAVX > 0, "requires some form of AVX");
8601 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8602 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8603 emit_int16(0x61, (0xC0 | encode));
8604 }
8605
8606 void Assembler::vpunpckhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8607 assert(UseAVX > 0, "requires some form of AVX");
8608 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8609 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8610 emit_int16(0x6A, (0xC0 | encode));
8611 }
8612
8613 void Assembler::vpunpckldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
8614 assert(UseAVX > 0, "requires some form of AVX");
8615 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
8616 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8617 emit_int16(0x62, (0xC0 | encode));
8618 }
8619
8620 // xmm/mem sourced byte/word/dword/qword replicate
8621 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8622 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8623 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8624 attributes.set_is_evex_instruction();
8625 attributes.set_embedded_opmask_register_specifier(mask);
8626 if (merge) {
8627 attributes.reset_is_clear_context();
8628 }
8629 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8630 emit_int16((unsigned char)0xFC, (0xC0 | encode));
8631 }
8632
8633 void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8634 InstructionMark im(this);
8635 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8636 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8637 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8638 attributes.set_is_evex_instruction();
8639 attributes.set_embedded_opmask_register_specifier(mask);
8640 if (merge) {
8641 attributes.reset_is_clear_context();
8642 }
8643 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8644 emit_int8((unsigned char)0xFC);
8645 emit_operand(dst, src);
8646 }
8647
8648 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8649 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8650 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8651 attributes.set_is_evex_instruction();
8652 attributes.set_embedded_opmask_register_specifier(mask);
8653 if (merge) {
8654 attributes.reset_is_clear_context();
8655 }
8656 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8657 emit_int16((unsigned char)0xFD, (0xC0 | encode));
8658 }
8659
8660 void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8661 InstructionMark im(this);
8662 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8663 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8664 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8665 attributes.set_is_evex_instruction();
8666 attributes.set_embedded_opmask_register_specifier(mask);
8667 if (merge) {
8668 attributes.reset_is_clear_context();
8669 }
8670 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8671 emit_int8((unsigned char)0xFD);
8672 emit_operand(dst, src);
8673 }
8674
8675 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8676 assert(VM_Version::supports_evex(), "");
8677 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8678 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8679 attributes.set_is_evex_instruction();
8680 attributes.set_embedded_opmask_register_specifier(mask);
8681 if (merge) {
8682 attributes.reset_is_clear_context();
8683 }
8684 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8685 emit_int16((unsigned char)0xFE, (0xC0 | encode));
8686 }
8687
8688 void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8689 InstructionMark im(this);
8690 assert(VM_Version::supports_evex(), "");
8691 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8692 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8693 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8694 attributes.set_is_evex_instruction();
8695 attributes.set_embedded_opmask_register_specifier(mask);
8696 if (merge) {
8697 attributes.reset_is_clear_context();
8698 }
8699 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8700 emit_int8((unsigned char)0xFE);
8701 emit_operand(dst, src);
8702 }
8703
8704 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8705 assert(VM_Version::supports_evex(), "");
8706 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8707 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8708 attributes.set_is_evex_instruction();
8709 attributes.set_embedded_opmask_register_specifier(mask);
8710 if (merge) {
8711 attributes.reset_is_clear_context();
8712 }
8713 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8714 emit_int16((unsigned char)0xD4, (0xC0 | encode));
8715 }
8716
8717 void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8718 InstructionMark im(this);
8719 assert(VM_Version::supports_evex(), "");
8720 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8721 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8722 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8723 attributes.set_is_evex_instruction();
8724 attributes.set_embedded_opmask_register_specifier(mask);
8725 if (merge) {
8726 attributes.reset_is_clear_context();
8727 }
8728 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8729 emit_int8((unsigned char)0xD4);
8730 emit_operand(dst, src);
8731 }
8732
8733 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8734 assert(VM_Version::supports_evex(), "");
8735 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8736 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8737 attributes.set_is_evex_instruction();
8738 attributes.set_embedded_opmask_register_specifier(mask);
8739 if (merge) {
8740 attributes.reset_is_clear_context();
8741 }
8742 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8743 emit_int16(0x58, (0xC0 | encode));
8744 }
8745
8746 void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8747 InstructionMark im(this);
8748 assert(VM_Version::supports_evex(), "");
8749 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8750 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8751 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8752 attributes.set_is_evex_instruction();
8753 attributes.set_embedded_opmask_register_specifier(mask);
8754 if (merge) {
8755 attributes.reset_is_clear_context();
8756 }
8757 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8758 emit_int8(0x58);
8759 emit_operand(dst, src);
8760 }
8761
8762 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8763 assert(VM_Version::supports_evex(), "");
8764 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8765 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8766 attributes.set_is_evex_instruction();
8767 attributes.set_embedded_opmask_register_specifier(mask);
8768 if (merge) {
8769 attributes.reset_is_clear_context();
8770 }
8771 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8772 emit_int16(0x58, (0xC0 | encode));
8773 }
8774
8775 void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8776 InstructionMark im(this);
8777 assert(VM_Version::supports_evex(), "");
8778 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8779 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8780 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8781 attributes.set_is_evex_instruction();
8782 attributes.set_embedded_opmask_register_specifier(mask);
8783 if (merge) {
8784 attributes.reset_is_clear_context();
8785 }
8786 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8787 emit_int8(0x58);
8788 emit_operand(dst, src);
8789 }
8790
8791 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8792 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8793 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8794 attributes.set_is_evex_instruction();
8795 attributes.set_embedded_opmask_register_specifier(mask);
8796 if (merge) {
8797 attributes.reset_is_clear_context();
8798 }
8799 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8800 emit_int16((unsigned char)0xF8, (0xC0 | encode));
8801 }
8802
8803 void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8804 InstructionMark im(this);
8805 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8806 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8807 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8808 attributes.set_is_evex_instruction();
8809 attributes.set_embedded_opmask_register_specifier(mask);
8810 if (merge) {
8811 attributes.reset_is_clear_context();
8812 }
8813 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8814 emit_int8((unsigned char)0xF8);
8815 emit_operand(dst, src);
8816 }
8817
8818 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8819 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8820 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8821 attributes.set_is_evex_instruction();
8822 attributes.set_embedded_opmask_register_specifier(mask);
8823 if (merge) {
8824 attributes.reset_is_clear_context();
8825 }
8826 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8827 emit_int16((unsigned char)0xF9, (0xC0 | encode));
8828 }
8829
8830 void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8831 InstructionMark im(this);
8832 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8833 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8834 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8835 attributes.set_is_evex_instruction();
8836 attributes.set_embedded_opmask_register_specifier(mask);
8837 if (merge) {
8838 attributes.reset_is_clear_context();
8839 }
8840 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8841 emit_int8((unsigned char)0xF9);
8842 emit_operand(dst, src);
8843 }
8844
8845 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8846 assert(VM_Version::supports_evex(), "");
8847 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8848 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8849 attributes.set_is_evex_instruction();
8850 attributes.set_embedded_opmask_register_specifier(mask);
8851 if (merge) {
8852 attributes.reset_is_clear_context();
8853 }
8854 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8855 emit_int16((unsigned char)0xFA, (0xC0 | encode));
8856 }
8857
8858 void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8859 InstructionMark im(this);
8860 assert(VM_Version::supports_evex(), "");
8861 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8862 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8863 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8864 attributes.set_is_evex_instruction();
8865 attributes.set_embedded_opmask_register_specifier(mask);
8866 if (merge) {
8867 attributes.reset_is_clear_context();
8868 }
8869 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8870 emit_int8((unsigned char)0xFA);
8871 emit_operand(dst, src);
8872 }
8873
8874 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8875 assert(VM_Version::supports_evex(), "");
8876 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8877 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8878 attributes.set_is_evex_instruction();
8879 attributes.set_embedded_opmask_register_specifier(mask);
8880 if (merge) {
8881 attributes.reset_is_clear_context();
8882 }
8883 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8884 emit_int16((unsigned char)0xFB, (0xC0 | encode));
8885 }
8886
8887 void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8888 InstructionMark im(this);
8889 assert(VM_Version::supports_evex(), "");
8890 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8891 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8892 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8893 attributes.set_is_evex_instruction();
8894 attributes.set_embedded_opmask_register_specifier(mask);
8895 if (merge) {
8896 attributes.reset_is_clear_context();
8897 }
8898 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8899 emit_int8((unsigned char)0xFB);
8900 emit_operand(dst, src);
8901 }
8902
8903 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8904 assert(VM_Version::supports_evex(), "");
8905 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8906 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8907 attributes.set_is_evex_instruction();
8908 attributes.set_embedded_opmask_register_specifier(mask);
8909 if (merge) {
8910 attributes.reset_is_clear_context();
8911 }
8912 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8913 emit_int16(0x5C, (0xC0 | encode));
8914 }
8915
8916 void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8917 InstructionMark im(this);
8918 assert(VM_Version::supports_evex(), "");
8919 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8920 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8921 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8922 attributes.set_is_evex_instruction();
8923 attributes.set_embedded_opmask_register_specifier(mask);
8924 if (merge) {
8925 attributes.reset_is_clear_context();
8926 }
8927 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
8928 emit_int8(0x5C);
8929 emit_operand(dst, src);
8930 }
8931
8932 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8933 assert(VM_Version::supports_evex(), "");
8934 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8935 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8936 attributes.set_is_evex_instruction();
8937 attributes.set_embedded_opmask_register_specifier(mask);
8938 if (merge) {
8939 attributes.reset_is_clear_context();
8940 }
8941 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8942 emit_int16(0x5C, (0xC0 | encode));
8943 }
8944
8945 void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8946 InstructionMark im(this);
8947 assert(VM_Version::supports_evex(), "");
8948 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8949 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8950 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8951 attributes.set_is_evex_instruction();
8952 attributes.set_embedded_opmask_register_specifier(mask);
8953 if (merge) {
8954 attributes.reset_is_clear_context();
8955 }
8956 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8957 emit_int8(0x5C);
8958 emit_operand(dst, src);
8959 }
8960
8961 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8962 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8963 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8964 attributes.set_is_evex_instruction();
8965 attributes.set_embedded_opmask_register_specifier(mask);
8966 if (merge) {
8967 attributes.reset_is_clear_context();
8968 }
8969 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8970 emit_int16((unsigned char)0xD5, (0xC0 | encode));
8971 }
8972
8973 void Assembler::evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
8974 InstructionMark im(this);
8975 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
8976 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8977 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
8978 attributes.set_is_evex_instruction();
8979 attributes.set_embedded_opmask_register_specifier(mask);
8980 if (merge) {
8981 attributes.reset_is_clear_context();
8982 }
8983 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8984 emit_int8((unsigned char)0xD5);
8985 emit_operand(dst, src);
8986 }
8987
8988 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
8989 assert(VM_Version::supports_evex(), "");
8990 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
8991 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
8992 attributes.set_is_evex_instruction();
8993 attributes.set_embedded_opmask_register_specifier(mask);
8994 if (merge) {
8995 attributes.reset_is_clear_context();
8996 }
8997 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8998 emit_int16(0x40, (0xC0 | encode));
8999 }
9000
9001 void Assembler::evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9002 InstructionMark im(this);
9003 assert(VM_Version::supports_evex(), "");
9004 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9005 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9006 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9007 attributes.set_is_evex_instruction();
9008 attributes.set_embedded_opmask_register_specifier(mask);
9009 if (merge) {
9010 attributes.reset_is_clear_context();
9011 }
9012 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9013 emit_int8(0x40);
9014 emit_operand(dst, src);
9015 }
9016
9017 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9018 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9019 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9020 attributes.set_is_evex_instruction();
9021 attributes.set_embedded_opmask_register_specifier(mask);
9022 if (merge) {
9023 attributes.reset_is_clear_context();
9024 }
9025 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9026 emit_int16(0x40, (0xC0 | encode));
9027 }
9028
9029 void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9030 InstructionMark im(this);
9031 assert(VM_Version::supports_avx512dq() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9032 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9033 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9034 attributes.set_is_evex_instruction();
9035 attributes.set_embedded_opmask_register_specifier(mask);
9036 if (merge) {
9037 attributes.reset_is_clear_context();
9038 }
9039 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9040 emit_int8(0x40);
9041 emit_operand(dst, src);
9042 }
9043
9044 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9045 assert(VM_Version::supports_evex(), "");
9046 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9047 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9048 attributes.set_is_evex_instruction();
9049 attributes.set_embedded_opmask_register_specifier(mask);
9050 if (merge) {
9051 attributes.reset_is_clear_context();
9052 }
9053 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9054 emit_int16(0x59, (0xC0 | encode));
9055 }
9056
9057 void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9058 InstructionMark im(this);
9059 assert(VM_Version::supports_evex(), "");
9060 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9061 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9062 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9063 attributes.set_is_evex_instruction();
9064 attributes.set_embedded_opmask_register_specifier(mask);
9065 if (merge) {
9066 attributes.reset_is_clear_context();
9067 }
9068 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9069 emit_int8(0x59);
9070 emit_operand(dst, src);
9071 }
9072
9073 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9074 assert(VM_Version::supports_evex(), "");
9075 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9076 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9077 attributes.set_is_evex_instruction();
9078 attributes.set_embedded_opmask_register_specifier(mask);
9079 if (merge) {
9080 attributes.reset_is_clear_context();
9081 }
9082 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9083 emit_int16(0x59, (0xC0 | encode));
9084 }
9085
9086 void Assembler::evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9087 InstructionMark im(this);
9088 assert(VM_Version::supports_evex(), "");
9089 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9090 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9091 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9092 attributes.set_is_evex_instruction();
9093 attributes.set_embedded_opmask_register_specifier(mask);
9094 if (merge) {
9095 attributes.reset_is_clear_context();
9096 }
9097 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9098 emit_int8(0x59);
9099 emit_operand(dst, src);
9100 }
9101
9102 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9103 assert(VM_Version::supports_evex(), "");
9104 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9105 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9106 attributes.set_is_evex_instruction();
9107 attributes.set_embedded_opmask_register_specifier(mask);
9108 if (merge) {
9109 attributes.reset_is_clear_context();
9110 }
9111 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9112 emit_int16(0x51, (0xC0 | encode));
9113 }
9114
9115 void Assembler::evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9116 InstructionMark im(this);
9117 assert(VM_Version::supports_evex(), "");
9118 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9119 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9120 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9121 attributes.set_is_evex_instruction();
9122 attributes.set_embedded_opmask_register_specifier(mask);
9123 if (merge) {
9124 attributes.reset_is_clear_context();
9125 }
9126 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9127 emit_int8(0x51);
9128 emit_operand(dst, src);
9129 }
9130
9131 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9132 assert(VM_Version::supports_evex(), "");
9133 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9134 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9135 attributes.set_is_evex_instruction();
9136 attributes.set_embedded_opmask_register_specifier(mask);
9137 if (merge) {
9138 attributes.reset_is_clear_context();
9139 }
9140 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9141 emit_int16(0x51, (0xC0 | encode));
9142 }
9143
9144 void Assembler::evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9145 InstructionMark im(this);
9146 assert(VM_Version::supports_evex(), "");
9147 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9148 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9149 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9150 attributes.set_is_evex_instruction();
9151 attributes.set_embedded_opmask_register_specifier(mask);
9152 if (merge) {
9153 attributes.reset_is_clear_context();
9154 }
9155 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9156 emit_int8(0x51);
9157 emit_operand(dst, src);
9158 }
9159
9160
9161 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9162 assert(VM_Version::supports_evex(), "");
9163 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9164 InstructionAttr attributes(vector_len,/* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9165 attributes.set_is_evex_instruction();
9166 attributes.set_embedded_opmask_register_specifier(mask);
9167 if (merge) {
9168 attributes.reset_is_clear_context();
9169 }
9170 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9171 emit_int16(0x5E, (0xC0 | encode));
9172 }
9173
9174 void Assembler::evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9175 InstructionMark im(this);
9176 assert(VM_Version::supports_evex(), "");
9177 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9178 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9179 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9180 attributes.set_is_evex_instruction();
9181 attributes.set_embedded_opmask_register_specifier(mask);
9182 if (merge) {
9183 attributes.reset_is_clear_context();
9184 }
9185 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
9186 emit_int8(0x5E);
9187 emit_operand(dst, src);
9188 }
9189
9190 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9191 assert(VM_Version::supports_evex(), "");
9192 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9193 InstructionAttr attributes(vector_len,/* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9194 attributes.set_is_evex_instruction();
9195 attributes.set_embedded_opmask_register_specifier(mask);
9196 if (merge) {
9197 attributes.reset_is_clear_context();
9198 }
9199 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9200 emit_int16(0x5E, (0xC0 | encode));
9201 }
9202
9203 void Assembler::evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9204 InstructionMark im(this);
9205 assert(VM_Version::supports_evex(), "");
9206 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9207 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9208 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9209 attributes.set_is_evex_instruction();
9210 attributes.set_embedded_opmask_register_specifier(mask);
9211 if (merge) {
9212 attributes.reset_is_clear_context();
9213 }
9214 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9215 emit_int8(0x5E);
9216 emit_operand(dst, src);
9217 }
9218
9219 void Assembler::evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9220 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9221 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9222 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9223 attributes.set_is_evex_instruction();
9224 attributes.set_embedded_opmask_register_specifier(mask);
9225 if (merge) {
9226 attributes.reset_is_clear_context();
9227 }
9228 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9229 emit_int16(0x1C, (0xC0 | encode));
9230 }
9231
9232
9233 void Assembler::evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9234 InstructionMark im(this);
9235 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9236 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9237 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9238 attributes.set_is_evex_instruction();
9239 attributes.set_embedded_opmask_register_specifier(mask);
9240 if (merge) {
9241 attributes.reset_is_clear_context();
9242 }
9243 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9244 emit_int8(0x1C);
9245 emit_operand(dst, src);
9246 }
9247
9248 void Assembler::evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9249 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9250 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9251 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9252 attributes.set_is_evex_instruction();
9253 attributes.set_embedded_opmask_register_specifier(mask);
9254 if (merge) {
9255 attributes.reset_is_clear_context();
9256 }
9257 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9258 emit_int16(0x1D, (0xC0 | encode));
9259 }
9260
9261
9262 void Assembler::evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9263 InstructionMark im(this);
9264 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9265 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9266 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9267 attributes.set_is_evex_instruction();
9268 attributes.set_embedded_opmask_register_specifier(mask);
9269 if (merge) {
9270 attributes.reset_is_clear_context();
9271 }
9272 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9273 emit_int8(0x1D);
9274 emit_operand(dst, src);
9275 }
9276
9277 void Assembler::evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9278 assert(VM_Version::supports_evex(), "");
9279 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9280 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9281 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9282 attributes.set_is_evex_instruction();
9283 attributes.set_embedded_opmask_register_specifier(mask);
9284 if (merge) {
9285 attributes.reset_is_clear_context();
9286 }
9287 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9288 emit_int16(0x1E, (0xC0 | encode));
9289 }
9290
9291
9292 void Assembler::evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9293 InstructionMark im(this);
9294 assert(VM_Version::supports_evex(), "");
9295 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9296 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9297 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9298 attributes.set_is_evex_instruction();
9299 attributes.set_embedded_opmask_register_specifier(mask);
9300 if (merge) {
9301 attributes.reset_is_clear_context();
9302 }
9303 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9304 emit_int8(0x1E);
9305 emit_operand(dst, src);
9306 }
9307
9308 void Assembler::evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
9309 assert(VM_Version::supports_evex(), "");
9310 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9311 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9312 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9313 attributes.set_is_evex_instruction();
9314 attributes.set_embedded_opmask_register_specifier(mask);
9315 if (merge) {
9316 attributes.reset_is_clear_context();
9317 }
9318 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9319 emit_int16(0x1F, (0xC0 | encode));
9320 }
9321
9322
9323 void Assembler::evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
9324 InstructionMark im(this);
9325 assert(VM_Version::supports_evex(), "");
9326 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9327 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9328 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9329 attributes.set_is_evex_instruction();
9330 attributes.set_embedded_opmask_register_specifier(mask);
9331 if (merge) {
9332 attributes.reset_is_clear_context();
9333 }
9334 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9335 emit_int8(0x1F);
9336 emit_operand(dst, src);
9337 }
9338
9339 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9340 assert(VM_Version::supports_evex(), "");
9341 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9342 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9343 attributes.set_is_evex_instruction();
9344 attributes.set_embedded_opmask_register_specifier(mask);
9345 if (merge) {
9346 attributes.reset_is_clear_context();
9347 }
9348 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9349 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9350 }
9351
9352 void Assembler::evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9353 InstructionMark im(this);
9354 assert(VM_Version::supports_evex(), "");
9355 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9356 InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9357 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9358 attributes.set_is_evex_instruction();
9359 attributes.set_embedded_opmask_register_specifier(mask);
9360 if (merge) {
9361 attributes.reset_is_clear_context();
9362 }
9363 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9364 emit_int8((unsigned char)0xA8);
9365 emit_operand(dst, src);
9366 }
9367
9368 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9369 assert(VM_Version::supports_evex(), "");
9370 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9371 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9372 attributes.set_is_evex_instruction();
9373 attributes.set_embedded_opmask_register_specifier(mask);
9374 if (merge) {
9375 attributes.reset_is_clear_context();
9376 }
9377 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9378 emit_int16((unsigned char)0xA8, (0xC0 | encode));
9379 }
9380
9381 void Assembler::evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9382 InstructionMark im(this);
9383 assert(VM_Version::supports_evex(), "");
9384 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9385 InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
9386 attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_32bit);
9387 attributes.set_is_evex_instruction();
9388 attributes.set_embedded_opmask_register_specifier(mask);
9389 if (merge) {
9390 attributes.reset_is_clear_context();
9391 }
9392 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9393 emit_int8((unsigned char)0xA8);
9394 emit_operand(dst, src);
9395 }
9396
9397 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9398 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9399 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9400 attributes.set_is_evex_instruction();
9401 attributes.set_embedded_opmask_register_specifier(mask);
9402 if (merge) {
9403 attributes.reset_is_clear_context();
9404 }
9405 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9406 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9407 }
9408
9409 void Assembler::evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9410 assert(VM_Version::supports_avx512_vbmi() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9411 InstructionMark im(this);
9412 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9413 attributes.set_is_evex_instruction();
9414 attributes.set_embedded_opmask_register_specifier(mask);
9415 if (merge) {
9416 attributes.reset_is_clear_context();
9417 }
9418 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9419 emit_int8((unsigned char)0x8D);
9420 emit_operand(dst, src);
9421 }
9422
9423 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9424 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9425 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9426 attributes.set_is_evex_instruction();
9427 attributes.set_embedded_opmask_register_specifier(mask);
9428 if (merge) {
9429 attributes.reset_is_clear_context();
9430 }
9431 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9432 emit_int16((unsigned char)0x8D, (0xC0 | encode));
9433 }
9434
9435 void Assembler::evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9436 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9437 InstructionMark im(this);
9438 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9439 attributes.set_is_evex_instruction();
9440 attributes.set_embedded_opmask_register_specifier(mask);
9441 if (merge) {
9442 attributes.reset_is_clear_context();
9443 }
9444 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9445 emit_int8((unsigned char)0x8D);
9446 emit_operand(dst, src);
9447 }
9448
9449 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9450 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9451 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9452 attributes.set_is_evex_instruction();
9453 attributes.set_embedded_opmask_register_specifier(mask);
9454 if (merge) {
9455 attributes.reset_is_clear_context();
9456 }
9457 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9458 emit_int16(0x36, (0xC0 | encode));
9459 }
9460
9461 void Assembler::evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9462 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9463 InstructionMark im(this);
9464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9465 attributes.set_is_evex_instruction();
9466 attributes.set_embedded_opmask_register_specifier(mask);
9467 if (merge) {
9468 attributes.reset_is_clear_context();
9469 }
9470 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9471 emit_int8(0x36);
9472 emit_operand(dst, src);
9473 }
9474
9475 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9476 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9477 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9478 attributes.set_is_evex_instruction();
9479 attributes.set_embedded_opmask_register_specifier(mask);
9480 if (merge) {
9481 attributes.reset_is_clear_context();
9482 }
9483 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9484 emit_int16(0x36, (0xC0 | encode));
9485 }
9486
9487 void Assembler::evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9488 assert(VM_Version::supports_evex() && vector_len > AVX_128bit, "");
9489 InstructionMark im(this);
9490 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9491 attributes.set_is_evex_instruction();
9492 attributes.set_embedded_opmask_register_specifier(mask);
9493 if (merge) {
9494 attributes.reset_is_clear_context();
9495 }
9496 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9497 emit_int8(0x36);
9498 emit_operand(dst, src);
9499 }
9500
9501 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9502 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9503 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9504 attributes.set_is_evex_instruction();
9505 attributes.set_embedded_opmask_register_specifier(mask);
9506 if (merge) {
9507 attributes.reset_is_clear_context();
9508 }
9509 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9510 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9511 }
9512
9513 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9514 assert(VM_Version::supports_evex(), "");
9515 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9516 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9517 attributes.set_is_evex_instruction();
9518 attributes.set_embedded_opmask_register_specifier(mask);
9519 if (merge) {
9520 attributes.reset_is_clear_context();
9521 }
9522 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9523 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9524 }
9525
9526 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9527 assert(VM_Version::supports_evex(), "");
9528 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9529 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9530 attributes.set_is_evex_instruction();
9531 attributes.set_embedded_opmask_register_specifier(mask);
9532 if (merge) {
9533 attributes.reset_is_clear_context();
9534 }
9535 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9536 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9537 }
9538
9539 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9540 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9541 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9542 attributes.set_is_evex_instruction();
9543 attributes.set_embedded_opmask_register_specifier(mask);
9544 if (merge) {
9545 attributes.reset_is_clear_context();
9546 }
9547 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9548 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9549 }
9550
9551 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9552 assert(VM_Version::supports_evex(), "");
9553 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9554 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9555 attributes.set_is_evex_instruction();
9556 attributes.set_embedded_opmask_register_specifier(mask);
9557 if (merge) {
9558 attributes.reset_is_clear_context();
9559 }
9560 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9561 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9562 }
9563
9564 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9565 assert(VM_Version::supports_evex(), "");
9566 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9567 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9568 attributes.set_is_evex_instruction();
9569 attributes.set_embedded_opmask_register_specifier(mask);
9570 if (merge) {
9571 attributes.reset_is_clear_context();
9572 }
9573 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9574 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
9575 }
9576
9577 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9578 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9580 attributes.set_is_evex_instruction();
9581 attributes.set_embedded_opmask_register_specifier(mask);
9582 if (merge) {
9583 attributes.reset_is_clear_context();
9584 }
9585 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9586 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
9587 }
9588
9589 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9590 assert(VM_Version::supports_evex(), "");
9591 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9592 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9593 attributes.set_is_evex_instruction();
9594 attributes.set_embedded_opmask_register_specifier(mask);
9595 if (merge) {
9596 attributes.reset_is_clear_context();
9597 }
9598 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9599 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9600 }
9601
9602 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
9603 assert(VM_Version::supports_evex(), "");
9604 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9605 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9606 attributes.set_is_evex_instruction();
9607 attributes.set_embedded_opmask_register_specifier(mask);
9608 if (merge) {
9609 attributes.reset_is_clear_context();
9610 }
9611 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9612 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
9613 }
9614
9615 void Assembler::evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9616 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9617 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9618 attributes.set_is_evex_instruction();
9619 attributes.set_embedded_opmask_register_specifier(mask);
9620 if (merge) {
9621 attributes.reset_is_clear_context();
9622 }
9623 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9624 emit_int16((unsigned char)0xF1, (0xC0 | encode));
9625 }
9626
9627 void Assembler::evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9628 assert(VM_Version::supports_evex(), "");
9629 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9630 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9631 attributes.set_is_evex_instruction();
9632 attributes.set_embedded_opmask_register_specifier(mask);
9633 if (merge) {
9634 attributes.reset_is_clear_context();
9635 }
9636 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9637 emit_int16((unsigned char)0xF2, (0xC0 | encode));
9638 }
9639
9640 void Assembler::evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9641 assert(VM_Version::supports_evex(), "");
9642 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9643 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9644 attributes.set_is_evex_instruction();
9645 attributes.set_embedded_opmask_register_specifier(mask);
9646 if (merge) {
9647 attributes.reset_is_clear_context();
9648 }
9649 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9650 emit_int16((unsigned char)0xF3, (0xC0 | encode));
9651 }
9652
9653 void Assembler::evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9654 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9655 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9656 attributes.set_is_evex_instruction();
9657 attributes.set_embedded_opmask_register_specifier(mask);
9658 if (merge) {
9659 attributes.reset_is_clear_context();
9660 }
9661 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9662 emit_int16((unsigned char)0xD1, (0xC0 | encode));
9663 }
9664
9665 void Assembler::evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9666 assert(VM_Version::supports_evex(), "");
9667 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9668 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9669 attributes.set_is_evex_instruction();
9670 attributes.set_embedded_opmask_register_specifier(mask);
9671 if (merge) {
9672 attributes.reset_is_clear_context();
9673 }
9674 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9675 emit_int16((unsigned char)0xD2, (0xC0 | encode));
9676 }
9677
9678 void Assembler::evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9679 assert(VM_Version::supports_evex(), "");
9680 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9681 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9682 attributes.set_is_evex_instruction();
9683 attributes.set_embedded_opmask_register_specifier(mask);
9684 if (merge) {
9685 attributes.reset_is_clear_context();
9686 }
9687 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9688 emit_int16((unsigned char)0xD3, (0xC0 | encode));
9689 }
9690
9691 void Assembler::evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9692 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9693 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9694 attributes.set_is_evex_instruction();
9695 attributes.set_embedded_opmask_register_specifier(mask);
9696 if (merge) {
9697 attributes.reset_is_clear_context();
9698 }
9699 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9700 emit_int16((unsigned char)0xE1, (0xC0 | encode));
9701 }
9702
9703 void Assembler::evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9704 assert(VM_Version::supports_evex(), "");
9705 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9706 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9707 attributes.set_is_evex_instruction();
9708 attributes.set_embedded_opmask_register_specifier(mask);
9709 if (merge) {
9710 attributes.reset_is_clear_context();
9711 }
9712 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9713 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9714 }
9715
9716 void Assembler::evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9717 assert(VM_Version::supports_evex(), "");
9718 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9719 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9720 attributes.set_is_evex_instruction();
9721 attributes.set_embedded_opmask_register_specifier(mask);
9722 if (merge) {
9723 attributes.reset_is_clear_context();
9724 }
9725 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9726 emit_int16((unsigned char)0xE2, (0xC0 | encode));
9727 }
9728
9729 void Assembler::evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9730 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9731 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9732 attributes.set_is_evex_instruction();
9733 attributes.set_embedded_opmask_register_specifier(mask);
9734 if (merge) {
9735 attributes.reset_is_clear_context();
9736 }
9737 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9738 emit_int16(0x12, (0xC0 | encode));
9739 }
9740
9741 void Assembler::evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9742 assert(VM_Version::supports_evex(), "");
9743 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9744 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9745 attributes.set_is_evex_instruction();
9746 attributes.set_embedded_opmask_register_specifier(mask);
9747 if (merge) {
9748 attributes.reset_is_clear_context();
9749 }
9750 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9751 emit_int16(0x47, (0xC0 | encode));
9752 }
9753
9754 void Assembler::evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9755 assert(VM_Version::supports_evex(), "");
9756 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9757 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9758 attributes.set_is_evex_instruction();
9759 attributes.set_embedded_opmask_register_specifier(mask);
9760 if (merge) {
9761 attributes.reset_is_clear_context();
9762 }
9763 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9764 emit_int16(0x47, (0xC0 | encode));
9765 }
9766
9767 void Assembler::evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9768 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9769 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9770 attributes.set_is_evex_instruction();
9771 attributes.set_embedded_opmask_register_specifier(mask);
9772 if (merge) {
9773 attributes.reset_is_clear_context();
9774 }
9775 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9776 emit_int16(0x10, (0xC0 | encode));
9777 }
9778
9779 void Assembler::evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9780 assert(VM_Version::supports_evex(), "");
9781 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9782 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9783 attributes.set_is_evex_instruction();
9784 attributes.set_embedded_opmask_register_specifier(mask);
9785 if (merge) {
9786 attributes.reset_is_clear_context();
9787 }
9788 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9789 emit_int16(0x45, (0xC0 | encode));
9790 }
9791
9792 void Assembler::evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9793 assert(VM_Version::supports_evex(), "");
9794 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9795 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9796 attributes.set_is_evex_instruction();
9797 attributes.set_embedded_opmask_register_specifier(mask);
9798 if (merge) {
9799 attributes.reset_is_clear_context();
9800 }
9801 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9802 emit_int16(0x45, (0xC0 | encode));
9803 }
9804
9805 void Assembler::evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9806 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9807 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9808 attributes.set_is_evex_instruction();
9809 attributes.set_embedded_opmask_register_specifier(mask);
9810 if (merge) {
9811 attributes.reset_is_clear_context();
9812 }
9813 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9814 emit_int16(0x11, (0xC0 | encode));
9815 }
9816
9817 void Assembler::evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9818 assert(VM_Version::supports_evex(), "");
9819 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9820 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9821 attributes.set_is_evex_instruction();
9822 attributes.set_embedded_opmask_register_specifier(mask);
9823 if (merge) {
9824 attributes.reset_is_clear_context();
9825 }
9826 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9827 emit_int16(0x46, (0xC0 | encode));
9828 }
9829
9830 void Assembler::evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9831 assert(VM_Version::supports_evex(), "");
9832 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9833 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9834 attributes.set_is_evex_instruction();
9835 attributes.set_embedded_opmask_register_specifier(mask);
9836 if (merge) {
9837 attributes.reset_is_clear_context();
9838 }
9839 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9840 emit_int16(0x46, (0xC0 | encode));
9841 }
9842
9843 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9844 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9845 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9846 attributes.set_is_evex_instruction();
9847 attributes.set_embedded_opmask_register_specifier(mask);
9848 if (merge) {
9849 attributes.reset_is_clear_context();
9850 }
9851 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9852 emit_int16(0x38, (0xC0 | encode));
9853 }
9854
9855 void Assembler::evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9856 assert(VM_Version::supports_avx512bw(), "");
9857 InstructionMark im(this);
9858 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9859 attributes.set_is_evex_instruction();
9860 attributes.set_embedded_opmask_register_specifier(mask);
9861 if (merge) {
9862 attributes.reset_is_clear_context();
9863 }
9864 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9865 emit_int8(0x38);
9866 emit_operand(dst, src);
9867 }
9868
9869 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9870 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9871 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9872 attributes.set_is_evex_instruction();
9873 attributes.set_embedded_opmask_register_specifier(mask);
9874 if (merge) {
9875 attributes.reset_is_clear_context();
9876 }
9877 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9878 emit_int16((unsigned char)0xEA, (0xC0 | encode));
9879 }
9880
9881 void Assembler::evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9882 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9883 InstructionMark im(this);
9884 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9885 attributes.set_is_evex_instruction();
9886 attributes.set_embedded_opmask_register_specifier(mask);
9887 if (merge) {
9888 attributes.reset_is_clear_context();
9889 }
9890 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9891 emit_int8((unsigned char)0xEA);
9892 emit_operand(dst, src);
9893 }
9894
9895 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9896 assert(VM_Version::supports_evex(), "");
9897 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9898 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9899 attributes.set_is_evex_instruction();
9900 attributes.set_embedded_opmask_register_specifier(mask);
9901 if (merge) {
9902 attributes.reset_is_clear_context();
9903 }
9904 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9905 emit_int16(0x39, (0xC0 | encode));
9906 }
9907
9908 void Assembler::evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9909 assert(VM_Version::supports_evex(), "");
9910 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9911 InstructionMark im(this);
9912 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9913 attributes.set_is_evex_instruction();
9914 attributes.set_embedded_opmask_register_specifier(mask);
9915 if (merge) {
9916 attributes.reset_is_clear_context();
9917 }
9918 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9919 emit_int8(0x39);
9920 emit_operand(dst, src);
9921 }
9922
9923 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9924 assert(VM_Version::supports_evex(), "");
9925 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9926 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9927 attributes.set_is_evex_instruction();
9928 attributes.set_embedded_opmask_register_specifier(mask);
9929 if (merge) {
9930 attributes.reset_is_clear_context();
9931 }
9932 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9933 emit_int16(0x39, (0xC0 | encode));
9934 }
9935
9936 void Assembler::evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9937 assert(VM_Version::supports_evex(), "");
9938 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
9939 InstructionMark im(this);
9940 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9941 attributes.set_is_evex_instruction();
9942 attributes.set_embedded_opmask_register_specifier(mask);
9943 if (merge) {
9944 attributes.reset_is_clear_context();
9945 }
9946 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9947 emit_int8(0x39);
9948 emit_operand(dst, src);
9949 }
9950
9951
9952 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9953 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9954 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9955 attributes.set_is_evex_instruction();
9956 attributes.set_embedded_opmask_register_specifier(mask);
9957 if (merge) {
9958 attributes.reset_is_clear_context();
9959 }
9960 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9961 emit_int16(0x3C, (0xC0 | encode));
9962 }
9963
9964 void Assembler::evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9965 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9966 InstructionMark im(this);
9967 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9968 attributes.set_is_evex_instruction();
9969 attributes.set_embedded_opmask_register_specifier(mask);
9970 if (merge) {
9971 attributes.reset_is_clear_context();
9972 }
9973 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
9974 emit_int8(0x3C);
9975 emit_operand(dst, src);
9976 }
9977
9978 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
9979 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9980 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9981 attributes.set_is_evex_instruction();
9982 attributes.set_embedded_opmask_register_specifier(mask);
9983 if (merge) {
9984 attributes.reset_is_clear_context();
9985 }
9986 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
9987 emit_int16((unsigned char)0xEE, (0xC0 | encode));
9988 }
9989
9990 void Assembler::evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
9991 assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
9992 InstructionMark im(this);
9993 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
9994 attributes.set_is_evex_instruction();
9995 attributes.set_embedded_opmask_register_specifier(mask);
9996 if (merge) {
9997 attributes.reset_is_clear_context();
9998 }
9999 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
10000 emit_int8((unsigned char)0xEE);
10001 emit_operand(dst, src);
10002 }
10003
10004 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10005 assert(VM_Version::supports_evex(), "");
10006 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10007 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10008 attributes.set_is_evex_instruction();
10009 attributes.set_embedded_opmask_register_specifier(mask);
10010 if (merge) {
10011 attributes.reset_is_clear_context();
10012 }
10013 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10014 emit_int16(0x3D, (0xC0 | encode));
10015 }
10016
10017 void Assembler::evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10018 assert(VM_Version::supports_evex(), "");
10019 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10020 InstructionMark im(this);
10021 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10022 attributes.set_is_evex_instruction();
10023 attributes.set_embedded_opmask_register_specifier(mask);
10024 if (merge) {
10025 attributes.reset_is_clear_context();
10026 }
10027 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10028 emit_int8(0x3D);
10029 emit_operand(dst, src);
10030 }
10031
10032 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10033 assert(VM_Version::supports_evex(), "");
10034 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10035 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10036 attributes.set_is_evex_instruction();
10037 attributes.set_embedded_opmask_register_specifier(mask);
10038 if (merge) {
10039 attributes.reset_is_clear_context();
10040 }
10041 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10042 emit_int16(0x3D, (0xC0 | encode));
10043 }
10044
10045 void Assembler::evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10046 assert(VM_Version::supports_evex(), "");
10047 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
10048 InstructionMark im(this);
10049 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10050 attributes.set_is_evex_instruction();
10051 attributes.set_embedded_opmask_register_specifier(mask);
10052 if (merge) {
10053 attributes.reset_is_clear_context();
10054 }
10055 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10056 emit_int8(0x3D);
10057 emit_operand(dst, src);
10058 }
10059
10060 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10061 assert(VM_Version::supports_evex(), "requires EVEX support");
10062 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10063 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10064 attributes.set_is_evex_instruction();
10065 attributes.set_embedded_opmask_register_specifier(mask);
10066 if (merge) {
10067 attributes.reset_is_clear_context();
10068 }
10069 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10070 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10071 }
10072
10073 void Assembler::evpternlogd(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10074 assert(VM_Version::supports_evex(), "requires EVEX support");
10075 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10076 assert(dst != xnoreg, "sanity");
10077 InstructionMark im(this);
10078 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10079 attributes.set_is_evex_instruction();
10080 attributes.set_embedded_opmask_register_specifier(mask);
10081 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10082 if (merge) {
10083 attributes.reset_is_clear_context();
10084 }
10085 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10086 emit_int8(0x25);
10087 emit_operand(dst, src3);
10088 emit_int8(imm8);
10089 }
10090
10091 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, int vector_len) {
10092 assert(VM_Version::supports_evex(), "requires EVEX support");
10093 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10094 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10095 attributes.set_is_evex_instruction();
10096 attributes.set_embedded_opmask_register_specifier(mask);
10097 if (merge) {
10098 attributes.reset_is_clear_context();
10099 }
10100 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10101 emit_int24(0x25, (unsigned char)(0xC0 | encode), imm8);
10102 }
10103
10104 void Assembler::evpternlogq(XMMRegister dst, int imm8, KRegister mask, XMMRegister src2, Address src3, bool merge, int vector_len) {
10105 assert(VM_Version::supports_evex(), "requires EVEX support");
10106 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
10107 assert(dst != xnoreg, "sanity");
10108 InstructionMark im(this);
10109 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10110 attributes.set_is_evex_instruction();
10111 attributes.set_embedded_opmask_register_specifier(mask);
10112 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
10113 if (merge) {
10114 attributes.reset_is_clear_context();
10115 }
10116 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10117 emit_int8(0x25);
10118 emit_operand(dst, src3);
10119 emit_int8(imm8);
10120 }
10121
10122 void Assembler::gf2p8affineqb(XMMRegister dst, XMMRegister src, int imm8) {
10123 assert(VM_Version::supports_gfni(), "");
10124 assert(VM_Version::supports_sse(), "");
10125 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
10126 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10127 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10128 }
10129
10130 void Assembler::vgf2p8affineqb(XMMRegister dst, XMMRegister src2, XMMRegister src3, int imm8, int vector_len) {
10131 assert(VM_Version::supports_gfni(), "requires GFNI support");
10132 assert(VM_Version::supports_sse(), "");
10133 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10134 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10135 emit_int24((unsigned char)0xCE, (unsigned char)(0xC0 | encode), imm8);
10136 }
10137
10138 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10139 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
10140 assert(UseAVX >= 2, "");
10141 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10142 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10143 emit_int16(0x58, (0xC0 | encode));
10144 }
10145
10146 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
10147 assert(VM_Version::supports_avx2(), "");
10148 assert(dst != xnoreg, "sanity");
10149 InstructionMark im(this);
10150 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10151 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10152 // swap src<->dst for encoding
10153 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10154 emit_int8(0x58);
10155 emit_operand(dst, src);
10156 }
10157
10158 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10159 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
10160 assert(VM_Version::supports_avx2(), "");
10161 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10162 attributes.set_rex_vex_w_reverted();
10163 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10164 emit_int16(0x59, (0xC0 | encode));
10165 }
10166
10167 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
10168 assert(VM_Version::supports_avx2(), "");
10169 assert(dst != xnoreg, "sanity");
10170 InstructionMark im(this);
10171 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10172 attributes.set_rex_vex_w_reverted();
10173 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10174 // swap src<->dst for encoding
10175 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10176 emit_int8(0x59);
10177 emit_operand(dst, src);
10178 }
10179
10180 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
10181 assert(vector_len != Assembler::AVX_128bit, "");
10182 assert(VM_Version::supports_evex(), "");
10183 assert(dst != xnoreg, "sanity");
10184 InstructionMark im(this);
10185 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10186 attributes.set_rex_vex_w_reverted();
10187 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10188 // swap src<->dst for encoding
10189 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10190 emit_int8(0x5A);
10191 emit_operand(dst, src);
10192 }
10193
10194 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
10195 assert(vector_len != Assembler::AVX_128bit, "");
10196 assert(VM_Version::supports_avx512dq(), "");
10197 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10198 attributes.set_rex_vex_w_reverted();
10199 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10200 emit_int16(0x5A, (0xC0 | encode));
10201 }
10202
10203 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
10204 assert(vector_len != Assembler::AVX_128bit, "");
10205 assert(VM_Version::supports_avx512dq(), "");
10206 assert(dst != xnoreg, "sanity");
10207 InstructionMark im(this);
10208 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10209 attributes.set_rex_vex_w_reverted();
10210 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
10211 // swap src<->dst for encoding
10212 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10213 emit_int8(0x5A);
10214 emit_operand(dst, src);
10215 }
10216
10217 // scalar single/double precision replicate
10218
10219 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
10220 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
10221 assert(VM_Version::supports_avx2(), "");
10222 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10223 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10224 emit_int16(0x18, (0xC0 | encode));
10225 }
10226
10227 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
10228 assert(VM_Version::supports_avx(), "");
10229 assert(dst != xnoreg, "sanity");
10230 InstructionMark im(this);
10231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10232 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10233 // swap src<->dst for encoding
10234 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10235 emit_int8(0x18);
10236 emit_operand(dst, src);
10237 }
10238
10239 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
10240 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
10241 assert(VM_Version::supports_avx2(), "");
10242 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10243 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10244 attributes.set_rex_vex_w_reverted();
10245 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10246 emit_int16(0x19, (0xC0 | encode));
10247 }
10248
10249 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
10250 assert(VM_Version::supports_avx(), "");
10251 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
10252 assert(dst != xnoreg, "sanity");
10253 InstructionMark im(this);
10254 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10255 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
10256 attributes.set_rex_vex_w_reverted();
10257 // swap src<->dst for encoding
10258 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10259 emit_int8(0x19);
10260 emit_operand(dst, src);
10261 }
10262
10263 void Assembler::vbroadcastf128(XMMRegister dst, Address src, int vector_len) {
10264 assert(VM_Version::supports_avx(), "");
10265 assert(vector_len == AVX_256bit, "");
10266 assert(dst != xnoreg, "sanity");
10267 InstructionMark im(this);
10268 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10269 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
10270 // swap src<->dst for encoding
10271 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10272 emit_int8(0x1A);
10273 emit_operand(dst, src);
10274 }
10275
10276 // gpr source broadcast forms
10277
10278 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10279 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
10280 assert(VM_Version::supports_avx512bw(), "");
10281 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10282 attributes.set_is_evex_instruction();
10283 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10284 emit_int16(0x7A, (0xC0 | encode));
10285 }
10286
10287 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
10288 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
10289 assert(VM_Version::supports_avx512bw(), "");
10290 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
10291 attributes.set_is_evex_instruction();
10292 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10293 emit_int16(0x7B, (0xC0 | encode));
10294 }
10295
10296 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
10297 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
10298 assert(VM_Version::supports_evex(), "");
10299 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10300 attributes.set_is_evex_instruction();
10301 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10302 emit_int16(0x7C, (0xC0 | encode));
10303 }
10304
10305 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
10306 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
10307 assert(VM_Version::supports_evex(), "");
10308 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10309 attributes.set_is_evex_instruction();
10310 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10311 emit_int16(0x7C, (0xC0 | encode));
10312 }
10313
10314 void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10315 assert(VM_Version::supports_avx2(), "");
10316 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10317 assert(dst != xnoreg, "sanity");
10318 assert(src.isxmmindex(),"expected to be xmm index");
10319 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10320 InstructionMark im(this);
10321 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10322 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10323 emit_int8((unsigned char)0x90);
10324 emit_operand(dst, src);
10325 }
10326
10327 void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10328 assert(VM_Version::supports_avx2(), "");
10329 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10330 assert(dst != xnoreg, "sanity");
10331 assert(src.isxmmindex(),"expected to be xmm index");
10332 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10333 InstructionMark im(this);
10334 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10335 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10336 emit_int8((unsigned char)0x90);
10337 emit_operand(dst, src);
10338 }
10339
10340 void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10341 assert(VM_Version::supports_avx2(), "");
10342 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10343 assert(dst != xnoreg, "sanity");
10344 assert(src.isxmmindex(),"expected to be xmm index");
10345 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10346 InstructionMark im(this);
10347 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10348 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10349 emit_int8((unsigned char)0x92);
10350 emit_operand(dst, src);
10351 }
10352
10353 void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len) {
10354 assert(VM_Version::supports_avx2(), "");
10355 assert(vector_len == Assembler::AVX_128bit || vector_len == Assembler::AVX_256bit, "");
10356 assert(dst != xnoreg, "sanity");
10357 assert(src.isxmmindex(),"expected to be xmm index");
10358 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10359 InstructionMark im(this);
10360 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
10361 vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10362 emit_int8((unsigned char)0x92);
10363 emit_operand(dst, src);
10364 }
10365 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10366 assert(VM_Version::supports_evex(), "");
10367 assert(dst != xnoreg, "sanity");
10368 assert(src.isxmmindex(),"expected to be xmm index");
10369 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10370 assert(mask != k0, "instruction will #UD if mask is in k0");
10371 InstructionMark im(this);
10372 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10373 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10374 attributes.reset_is_clear_context();
10375 attributes.set_embedded_opmask_register_specifier(mask);
10376 attributes.set_is_evex_instruction();
10377 // swap src<->dst for encoding
10378 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10379 emit_int8((unsigned char)0x90);
10380 emit_operand(dst, src);
10381 }
10382
10383 void Assembler::evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10384 assert(VM_Version::supports_evex(), "");
10385 assert(dst != xnoreg, "sanity");
10386 assert(src.isxmmindex(),"expected to be xmm index");
10387 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10388 assert(mask != k0, "instruction will #UD if mask is in k0");
10389 InstructionMark im(this);
10390 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10391 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10392 attributes.reset_is_clear_context();
10393 attributes.set_embedded_opmask_register_specifier(mask);
10394 attributes.set_is_evex_instruction();
10395 // swap src<->dst for encoding
10396 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10397 emit_int8((unsigned char)0x90);
10398 emit_operand(dst, src);
10399 }
10400
10401 void Assembler::evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10402 assert(VM_Version::supports_evex(), "");
10403 assert(dst != xnoreg, "sanity");
10404 assert(src.isxmmindex(),"expected to be xmm index");
10405 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10406 assert(mask != k0, "instruction will #UD if mask is in k0");
10407 InstructionMark im(this);
10408 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10409 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10410 attributes.reset_is_clear_context();
10411 attributes.set_embedded_opmask_register_specifier(mask);
10412 attributes.set_is_evex_instruction();
10413 // swap src<->dst for encoding
10414 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10415 emit_int8((unsigned char)0x92);
10416 emit_operand(dst, src);
10417 }
10418
10419 void Assembler::evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len) {
10420 assert(VM_Version::supports_evex(), "");
10421 assert(dst != xnoreg, "sanity");
10422 assert(src.isxmmindex(),"expected to be xmm index");
10423 assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
10424 assert(mask != k0, "instruction will #UD if mask is in k0");
10425 InstructionMark im(this);
10426 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10427 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10428 attributes.reset_is_clear_context();
10429 attributes.set_embedded_opmask_register_specifier(mask);
10430 attributes.set_is_evex_instruction();
10431 // swap src<->dst for encoding
10432 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10433 emit_int8((unsigned char)0x92);
10434 emit_operand(dst, src);
10435 }
10436
10437 void Assembler::evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10438 assert(VM_Version::supports_evex(), "");
10439 assert(mask != k0, "instruction will #UD if mask is in k0");
10440 InstructionMark im(this);
10441 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10442 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10443 attributes.reset_is_clear_context();
10444 attributes.set_embedded_opmask_register_specifier(mask);
10445 attributes.set_is_evex_instruction();
10446 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10447 emit_int8((unsigned char)0xA0);
10448 emit_operand(src, dst);
10449 }
10450
10451 void Assembler::evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10452 assert(VM_Version::supports_evex(), "");
10453 assert(mask != k0, "instruction will #UD if mask is in k0");
10454 InstructionMark im(this);
10455 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10456 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10457 attributes.reset_is_clear_context();
10458 attributes.set_embedded_opmask_register_specifier(mask);
10459 attributes.set_is_evex_instruction();
10460 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10461 emit_int8((unsigned char)0xA0);
10462 emit_operand(src, dst);
10463 }
10464
10465 void Assembler::evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10466 assert(VM_Version::supports_evex(), "");
10467 assert(mask != k0, "instruction will #UD if mask is in k0");
10468 InstructionMark im(this);
10469 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10470 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10471 attributes.reset_is_clear_context();
10472 attributes.set_embedded_opmask_register_specifier(mask);
10473 attributes.set_is_evex_instruction();
10474 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10475 emit_int8((unsigned char)0xA2);
10476 emit_operand(src, dst);
10477 }
10478
10479 void Assembler::evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len) {
10480 assert(VM_Version::supports_evex(), "");
10481 assert(mask != k0, "instruction will #UD if mask is in k0");
10482 InstructionMark im(this);
10483 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
10484 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
10485 attributes.reset_is_clear_context();
10486 attributes.set_embedded_opmask_register_specifier(mask);
10487 attributes.set_is_evex_instruction();
10488 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
10489 emit_int8((unsigned char)0xA2);
10490 emit_operand(src, dst);
10491 }
10492 // Carry-Less Multiplication Quadword
10493 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
10494 assert(VM_Version::supports_clmul(), "");
10495 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10496 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10497 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10498 }
10499
10500 // Carry-Less Multiplication Quadword
10501 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
10502 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
10503 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
10504 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10505 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10506 }
10507
10508 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
10509 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
10510 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
10511 attributes.set_is_evex_instruction();
10512 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10513 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
10514 }
10515
10516 void Assembler::vzeroupper_uncached() {
10517 if (VM_Version::supports_vzeroupper()) {
10518 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
10519 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
10520 emit_int8(0x77);
10521 }
10522 }
10523
10524 void Assembler::vfpclassss(KRegister kdst, XMMRegister src, uint8_t imm8) {
10525 // Encoding: EVEX.LIG.66.0F3A.W0 67 /r ib
10526 assert(VM_Version::supports_evex(), "");
10527 assert(VM_Version::supports_avx512dq(), "");
10528 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10529 attributes.set_is_evex_instruction();
10530 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10531 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10532 }
10533
10534 void Assembler::vfpclasssd(KRegister kdst, XMMRegister src, uint8_t imm8) {
10535 // Encoding: EVEX.LIG.66.0F3A.W1 67 /r ib
10536 assert(VM_Version::supports_evex(), "");
10537 assert(VM_Version::supports_avx512dq(), "");
10538 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
10539 attributes.set_is_evex_instruction();
10540 int encode = vex_prefix_and_encode(kdst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
10541 emit_int24((unsigned char)0x67, (unsigned char)(0xC0 | encode), imm8);
10542 }
10543
10544 void Assembler::fld_x(Address adr) {
10545 InstructionMark im(this);
10546 emit_int8((unsigned char)0xDB);
10547 emit_operand32(rbp, adr);
10548 }
10549
10550 void Assembler::fstp_x(Address adr) {
10551 InstructionMark im(this);
10552 emit_int8((unsigned char)0xDB);
10553 emit_operand32(rdi, adr);
10554 }
10555
10556 void Assembler::emit_operand32(Register reg, Address adr) {
10557 assert(reg->encoding() < 8, "no extended registers");
10558 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
10559 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
10560 adr._rspec);
10561 }
10562
10563 #ifndef _LP64
10564 // 32bit only pieces of the assembler
10565
10566 void Assembler::emms() {
10567 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
10568 emit_int16(0x0F, 0x77);
10569 }
10570
10571 void Assembler::vzeroupper() {
10572 vzeroupper_uncached();
10573 }
10574
10575 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
10576 // NO PREFIX AS NEVER 64BIT
10577 InstructionMark im(this);
10578 emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
10579 emit_data(imm32, rspec, 0);
10580 }
10581
10582 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
10583 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
10584 InstructionMark im(this);
10585 emit_int8((unsigned char)0x81);
10586 emit_operand(rdi, src1);
10587 emit_data(imm32, rspec, 0);
10588 }
10589
10590 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
10591 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
10592 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
10593 void Assembler::cmpxchg8(Address adr) {
10594 InstructionMark im(this);
10595 emit_int16(0x0F, (unsigned char)0xC7);
10596 emit_operand(rcx, adr);
10597 }
10598
10599 void Assembler::decl(Register dst) {
10600 // Don't use it directly. Use MacroAssembler::decrementl() instead.
10601 emit_int8(0x48 | dst->encoding());
10602 }
10603
10604 // 64bit doesn't use the x87
10605
10606 void Assembler::emit_farith(int b1, int b2, int i) {
10607 assert(isByte(b1) && isByte(b2), "wrong opcode");
10608 assert(0 <= i && i < 8, "illegal stack offset");
10609 emit_int16(b1, b2 + i);
10610 }
10611
10612 void Assembler::fabs() {
10613 emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
10614 }
10615
10616 void Assembler::fadd(int i) {
10617 emit_farith(0xD8, 0xC0, i);
10618 }
10619
10620 void Assembler::fadd_d(Address src) {
10621 InstructionMark im(this);
10622 emit_int8((unsigned char)0xDC);
10623 emit_operand32(rax, src);
10624 }
10625
10626 void Assembler::fadd_s(Address src) {
10627 InstructionMark im(this);
10628 emit_int8((unsigned char)0xD8);
10629 emit_operand32(rax, src);
10630 }
10631
10632 void Assembler::fadda(int i) {
10633 emit_farith(0xDC, 0xC0, i);
10634 }
10635
10636 void Assembler::faddp(int i) {
10637 emit_farith(0xDE, 0xC0, i);
10638 }
10639
10640 void Assembler::fchs() {
10641 emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
10642 }
10643
10644 void Assembler::fcom(int i) {
10645 emit_farith(0xD8, 0xD0, i);
10646 }
10647
10648 void Assembler::fcomp(int i) {
10649 emit_farith(0xD8, 0xD8, i);
10650 }
10651
10652 void Assembler::fcomp_d(Address src) {
10653 InstructionMark im(this);
10654 emit_int8((unsigned char)0xDC);
10655 emit_operand32(rbx, src);
10656 }
10657
10658 void Assembler::fcomp_s(Address src) {
10659 InstructionMark im(this);
10660 emit_int8((unsigned char)0xD8);
10661 emit_operand32(rbx, src);
10662 }
10663
10664 void Assembler::fcompp() {
10665 emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
10666 }
10667
10668 void Assembler::fcos() {
10669 emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
10670 }
10671
10672 void Assembler::fdecstp() {
10673 emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
10674 }
10675
10676 void Assembler::fdiv(int i) {
10677 emit_farith(0xD8, 0xF0, i);
10678 }
10679
10680 void Assembler::fdiv_d(Address src) {
10681 InstructionMark im(this);
10682 emit_int8((unsigned char)0xDC);
10683 emit_operand32(rsi, src);
10684 }
10685
10686 void Assembler::fdiv_s(Address src) {
10687 InstructionMark im(this);
10688 emit_int8((unsigned char)0xD8);
10689 emit_operand32(rsi, src);
10690 }
10691
10692 void Assembler::fdiva(int i) {
10693 emit_farith(0xDC, 0xF8, i);
10694 }
10695
10696 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
10697 // is erroneous for some of the floating-point instructions below.
10698
10699 void Assembler::fdivp(int i) {
10700 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
10701 }
10702
10703 void Assembler::fdivr(int i) {
10704 emit_farith(0xD8, 0xF8, i);
10705 }
10706
10707 void Assembler::fdivr_d(Address src) {
10708 InstructionMark im(this);
10709 emit_int8((unsigned char)0xDC);
10710 emit_operand32(rdi, src);
10711 }
10712
10713 void Assembler::fdivr_s(Address src) {
10714 InstructionMark im(this);
10715 emit_int8((unsigned char)0xD8);
10716 emit_operand32(rdi, src);
10717 }
10718
10719 void Assembler::fdivra(int i) {
10720 emit_farith(0xDC, 0xF0, i);
10721 }
10722
10723 void Assembler::fdivrp(int i) {
10724 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
10725 }
10726
10727 void Assembler::ffree(int i) {
10728 emit_farith(0xDD, 0xC0, i);
10729 }
10730
10731 void Assembler::fild_d(Address adr) {
10732 InstructionMark im(this);
10733 emit_int8((unsigned char)0xDF);
10734 emit_operand32(rbp, adr);
10735 }
10736
10737 void Assembler::fild_s(Address adr) {
10738 InstructionMark im(this);
10739 emit_int8((unsigned char)0xDB);
10740 emit_operand32(rax, adr);
10741 }
10742
10743 void Assembler::fincstp() {
10744 emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
10745 }
10746
10747 void Assembler::finit() {
10748 emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
10749 }
10750
10751 void Assembler::fist_s(Address adr) {
10752 InstructionMark im(this);
10753 emit_int8((unsigned char)0xDB);
10754 emit_operand32(rdx, adr);
10755 }
10756
10757 void Assembler::fistp_d(Address adr) {
10758 InstructionMark im(this);
10759 emit_int8((unsigned char)0xDF);
10760 emit_operand32(rdi, adr);
10761 }
10762
10763 void Assembler::fistp_s(Address adr) {
10764 InstructionMark im(this);
10765 emit_int8((unsigned char)0xDB);
10766 emit_operand32(rbx, adr);
10767 }
10768
10769 void Assembler::fld1() {
10770 emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
10771 }
10772
10773 void Assembler::fld_d(Address adr) {
10774 InstructionMark im(this);
10775 emit_int8((unsigned char)0xDD);
10776 emit_operand32(rax, adr);
10777 }
10778
10779 void Assembler::fld_s(Address adr) {
10780 InstructionMark im(this);
10781 emit_int8((unsigned char)0xD9);
10782 emit_operand32(rax, adr);
10783 }
10784
10785
10786 void Assembler::fld_s(int index) {
10787 emit_farith(0xD9, 0xC0, index);
10788 }
10789
10790 void Assembler::fldcw(Address src) {
10791 InstructionMark im(this);
10792 emit_int8((unsigned char)0xD9);
10793 emit_operand32(rbp, src);
10794 }
10795
10796 void Assembler::fldenv(Address src) {
10797 InstructionMark im(this);
10798 emit_int8((unsigned char)0xD9);
10799 emit_operand32(rsp, src);
10800 }
10801
10802 void Assembler::fldlg2() {
10803 emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
10804 }
10805
10806 void Assembler::fldln2() {
10807 emit_int16((unsigned char)0xD9, (unsigned char)0xED);
10808 }
10809
10810 void Assembler::fldz() {
10811 emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
10812 }
10813
10814 void Assembler::flog() {
10815 fldln2();
10816 fxch();
10817 fyl2x();
10818 }
10819
10820 void Assembler::flog10() {
10821 fldlg2();
10822 fxch();
10823 fyl2x();
10824 }
10825
10826 void Assembler::fmul(int i) {
10827 emit_farith(0xD8, 0xC8, i);
10828 }
10829
10830 void Assembler::fmul_d(Address src) {
10831 InstructionMark im(this);
10832 emit_int8((unsigned char)0xDC);
10833 emit_operand32(rcx, src);
10834 }
10835
10836 void Assembler::fmul_s(Address src) {
10837 InstructionMark im(this);
10838 emit_int8((unsigned char)0xD8);
10839 emit_operand32(rcx, src);
10840 }
10841
10842 void Assembler::fmula(int i) {
10843 emit_farith(0xDC, 0xC8, i);
10844 }
10845
10846 void Assembler::fmulp(int i) {
10847 emit_farith(0xDE, 0xC8, i);
10848 }
10849
10850 void Assembler::fnsave(Address dst) {
10851 InstructionMark im(this);
10852 emit_int8((unsigned char)0xDD);
10853 emit_operand32(rsi, dst);
10854 }
10855
10856 void Assembler::fnstcw(Address src) {
10857 InstructionMark im(this);
10858 emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
10859 emit_operand32(rdi, src);
10860 }
10861
10862 void Assembler::fnstsw_ax() {
10863 emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
10864 }
10865
10866 void Assembler::fprem() {
10867 emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
10868 }
10869
10870 void Assembler::fprem1() {
10871 emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
10872 }
10873
10874 void Assembler::frstor(Address src) {
10875 InstructionMark im(this);
10876 emit_int8((unsigned char)0xDD);
10877 emit_operand32(rsp, src);
10878 }
10879
10880 void Assembler::fsin() {
10881 emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
10882 }
10883
10884 void Assembler::fsqrt() {
10885 emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
10886 }
10887
10888 void Assembler::fst_d(Address adr) {
10889 InstructionMark im(this);
10890 emit_int8((unsigned char)0xDD);
10891 emit_operand32(rdx, adr);
10892 }
10893
10894 void Assembler::fst_s(Address adr) {
10895 InstructionMark im(this);
10896 emit_int8((unsigned char)0xD9);
10897 emit_operand32(rdx, adr);
10898 }
10899
10900 void Assembler::fstp_d(Address adr) {
10901 InstructionMark im(this);
10902 emit_int8((unsigned char)0xDD);
10903 emit_operand32(rbx, adr);
10904 }
10905
10906 void Assembler::fstp_d(int index) {
10907 emit_farith(0xDD, 0xD8, index);
10908 }
10909
10910 void Assembler::fstp_s(Address adr) {
10911 InstructionMark im(this);
10912 emit_int8((unsigned char)0xD9);
10913 emit_operand32(rbx, adr);
10914 }
10915
10916 void Assembler::fsub(int i) {
10917 emit_farith(0xD8, 0xE0, i);
10918 }
10919
10920 void Assembler::fsub_d(Address src) {
10921 InstructionMark im(this);
10922 emit_int8((unsigned char)0xDC);
10923 emit_operand32(rsp, src);
10924 }
10925
10926 void Assembler::fsub_s(Address src) {
10927 InstructionMark im(this);
10928 emit_int8((unsigned char)0xD8);
10929 emit_operand32(rsp, src);
10930 }
10931
10932 void Assembler::fsuba(int i) {
10933 emit_farith(0xDC, 0xE8, i);
10934 }
10935
10936 void Assembler::fsubp(int i) {
10937 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
10938 }
10939
10940 void Assembler::fsubr(int i) {
10941 emit_farith(0xD8, 0xE8, i);
10942 }
10943
10944 void Assembler::fsubr_d(Address src) {
10945 InstructionMark im(this);
10946 emit_int8((unsigned char)0xDC);
10947 emit_operand32(rbp, src);
10948 }
10949
10950 void Assembler::fsubr_s(Address src) {
10951 InstructionMark im(this);
10952 emit_int8((unsigned char)0xD8);
10953 emit_operand32(rbp, src);
10954 }
10955
10956 void Assembler::fsubra(int i) {
10957 emit_farith(0xDC, 0xE0, i);
10958 }
10959
10960 void Assembler::fsubrp(int i) {
10961 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
10962 }
10963
10964 void Assembler::ftan() {
10965 emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
10966 }
10967
10968 void Assembler::ftst() {
10969 emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
10970 }
10971
10972 void Assembler::fucomi(int i) {
10973 // make sure the instruction is supported (introduced for P6, together with cmov)
10974 guarantee(VM_Version::supports_cmov(), "illegal instruction");
10975 emit_farith(0xDB, 0xE8, i);
10976 }
10977
10978 void Assembler::fucomip(int i) {
10979 // make sure the instruction is supported (introduced for P6, together with cmov)
10980 guarantee(VM_Version::supports_cmov(), "illegal instruction");
10981 emit_farith(0xDF, 0xE8, i);
10982 }
10983
10984 void Assembler::fwait() {
10985 emit_int8((unsigned char)0x9B);
10986 }
10987
10988 void Assembler::fxch(int i) {
10989 emit_farith(0xD9, 0xC8, i);
10990 }
10991
10992 void Assembler::fyl2x() {
10993 emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
10994 }
10995
10996 void Assembler::frndint() {
10997 emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
10998 }
10999
11000 void Assembler::f2xm1() {
11001 emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
11002 }
11003
11004 void Assembler::fldl2e() {
11005 emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
11006 }
11007 #endif // !_LP64
11008
11009 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
11010 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
11011 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
11012 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
11013
11014 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
11015 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11016 if (pre > 0) {
11017 emit_int8(simd_pre[pre]);
11018 }
11019 if (rex_w) {
11020 prefixq(adr, xreg);
11021 } else {
11022 prefix(adr, xreg);
11023 }
11024 if (opc > 0) {
11025 emit_int8(0x0F);
11026 int opc2 = simd_opc[opc];
11027 if (opc2 > 0) {
11028 emit_int8(opc2);
11029 }
11030 }
11031 }
11032
11033 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
11034 if (pre > 0) {
11035 emit_int8(simd_pre[pre]);
11036 }
11037 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
11038 if (opc > 0) {
11039 emit_int8(0x0F);
11040 int opc2 = simd_opc[opc];
11041 if (opc2 > 0) {
11042 emit_int8(opc2);
11043 }
11044 }
11045 return encode;
11046 }
11047
11048
11049 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
11050 int vector_len = _attributes->get_vector_len();
11051 bool vex_w = _attributes->is_rex_vex_w();
11052 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
11053 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
11054 byte1 = (~byte1) & 0xE0;
11055 byte1 |= opc;
11056
11057 int byte2 = ((~nds_enc) & 0xf) << 3;
11058 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
11059
11060 emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
11061 } else {
11062 int byte1 = vex_r ? VEX_R : 0;
11063 byte1 = (~byte1) & 0x80;
11064 byte1 |= ((~nds_enc) & 0xf) << 3;
11065 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
11066 emit_int16((unsigned char)VEX_2bytes, byte1);
11067 }
11068 }
11069
11070 // This is a 4 byte encoding
11071 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
11072 // EVEX 0x62 prefix
11073 // byte1 = EVEX_4bytes;
11074
11075 bool vex_w = _attributes->is_rex_vex_w();
11076 int evex_encoding = (vex_w ? VEX_W : 0);
11077 // EVEX.b is not currently used for broadcast of single element or data rounding modes
11078 _attributes->set_evex_encoding(evex_encoding);
11079
11080 // P0: byte 2, initialized to RXBR`00mm
11081 // instead of not'd
11082 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
11083 byte2 = (~byte2) & 0xF0;
11084 // confine opc opcode extensions in mm bits to lower two bits
11085 // of form {0F, 0F_38, 0F_3A}
11086 byte2 |= opc;
11087
11088 // P1: byte 3 as Wvvvv1pp
11089 int byte3 = ((~nds_enc) & 0xf) << 3;
11090 // p[10] is always 1
11091 byte3 |= EVEX_F;
11092 byte3 |= (vex_w & 1) << 7;
11093 // confine pre opcode extensions in pp bits to lower two bits
11094 // of form {66, F3, F2}
11095 byte3 |= pre;
11096
11097 // P2: byte 4 as zL'Lbv'aaa
11098 // kregs are implemented in the low 3 bits as aaa
11099 int byte4 = (_attributes->is_no_reg_mask()) ?
11100 0 :
11101 _attributes->get_embedded_opmask_register_specifier();
11102 // EVEX.v` for extending EVEX.vvvv or VIDX
11103 byte4 |= (evex_v ? 0: EVEX_V);
11104 // third EXEC.b for broadcast actions
11105 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
11106 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
11107 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
11108 // last is EVEX.z for zero/merge actions
11109 if (_attributes->is_no_reg_mask() == false &&
11110 _attributes->get_embedded_opmask_register_specifier() != 0) {
11111 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
11112 }
11113
11114 emit_int32(EVEX_4bytes, byte2, byte3, byte4);
11115 }
11116
11117 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11118 bool vex_r = (xreg_enc & 8) == 8;
11119 bool vex_b = adr.base_needs_rex();
11120 bool vex_x;
11121 if (adr.isxmmindex()) {
11122 vex_x = adr.xmmindex_needs_rex();
11123 } else {
11124 vex_x = adr.index_needs_rex();
11125 }
11126 set_attributes(attributes);
11127 attributes->set_current_assembler(this);
11128
11129 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11130 // is allowed in legacy mode and has resources which will fit in it.
11131 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11132 if (!attributes->is_legacy_mode()) {
11133 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11134 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
11135 attributes->set_is_legacy_mode();
11136 }
11137 }
11138 }
11139
11140 if (UseAVX > 2) {
11141 assert(((!attributes->uses_vl()) ||
11142 (attributes->get_vector_len() == AVX_512bit) ||
11143 (!_legacy_mode_vl) ||
11144 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11145 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11146 }
11147
11148 clear_managed();
11149 if (UseAVX > 2 && !attributes->is_legacy_mode())
11150 {
11151 bool evex_r = (xreg_enc >= 16);
11152 bool evex_v;
11153 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
11154 if (adr.isxmmindex()) {
11155 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
11156 } else {
11157 evex_v = (nds_enc >= 16);
11158 }
11159 attributes->set_is_evex_instruction();
11160 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11161 } else {
11162 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11163 attributes->set_rex_vex_w(false);
11164 }
11165 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11166 }
11167 }
11168
11169 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
11170 bool vex_r = (dst_enc & 8) == 8;
11171 bool vex_b = (src_enc & 8) == 8;
11172 bool vex_x = false;
11173 set_attributes(attributes);
11174 attributes->set_current_assembler(this);
11175
11176 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
11177 // is allowed in legacy mode and has resources which will fit in it.
11178 // Pure EVEX instructions will have is_evex_instruction set in their definition.
11179 if (!attributes->is_legacy_mode()) {
11180 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
11181 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
11182 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
11183 attributes->set_is_legacy_mode();
11184 }
11185 }
11186 }
11187
11188 if (UseAVX > 2) {
11189 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
11190 // Instruction with uses_vl true are vector instructions
11191 // All the vector instructions with AVX_512bit length can have legacy_mode as false
11192 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
11193 // Rest all should have legacy_mode set as true
11194 assert(((!attributes->uses_vl()) ||
11195 (attributes->get_vector_len() == AVX_512bit) ||
11196 (!_legacy_mode_vl) ||
11197 (attributes->is_legacy_mode())),"XMM register should be 0-15");
11198 // Instruction with legacy_mode true should have dst, nds and src < 15
11199 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
11200 }
11201
11202 clear_managed();
11203 if (UseAVX > 2 && !attributes->is_legacy_mode())
11204 {
11205 bool evex_r = (dst_enc >= 16);
11206 bool evex_v = (nds_enc >= 16);
11207 // can use vex_x as bank extender on rm encoding
11208 vex_x = (src_enc >= 16);
11209 attributes->set_is_evex_instruction();
11210 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
11211 } else {
11212 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
11213 attributes->set_rex_vex_w(false);
11214 }
11215 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
11216 }
11217
11218 // return modrm byte components for operands
11219 return (((dst_enc & 7) << 3) | (src_enc & 7));
11220 }
11221
11222
11223 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
11224 VexOpcode opc, InstructionAttr *attributes) {
11225 if (UseAVX > 0) {
11226 int xreg_enc = xreg->encoding();
11227 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11228 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
11229 } else {
11230 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
11231 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
11232 }
11233 }
11234
11235 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
11236 VexOpcode opc, InstructionAttr *attributes) {
11237 int dst_enc = dst->encoding();
11238 int src_enc = src->encoding();
11239 if (UseAVX > 0) {
11240 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
11241 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
11242 } else {
11243 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
11244 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
11245 }
11246 }
11247
11248 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11249 assert(VM_Version::supports_avx(), "");
11250 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11251 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11252 emit_int16(0x5F, (0xC0 | encode));
11253 }
11254
11255 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11256 assert(VM_Version::supports_avx(), "");
11257 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11258 attributes.set_rex_vex_w_reverted();
11259 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11260 emit_int16(0x5F, (0xC0 | encode));
11261 }
11262
11263 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11264 assert(VM_Version::supports_avx(), "");
11265 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11266 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
11267 emit_int16(0x5D, (0xC0 | encode));
11268 }
11269
11270 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
11271 assert(VM_Version::supports_avx(), "");
11272 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
11273 attributes.set_rex_vex_w_reverted();
11274 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
11275 emit_int16(0x5D, (0xC0 | encode));
11276 }
11277
11278 void Assembler::vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
11279 assert(VM_Version::supports_avx(), "");
11280 assert(vector_len <= AVX_256bit, "");
11281 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11282 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11283 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
11284 }
11285
11286 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11287 assert(VM_Version::supports_avx(), "");
11288 assert(vector_len <= AVX_256bit, "");
11289 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11290 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11291 int src2_enc = src2->encoding();
11292 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
11293 }
11294
11295 void Assembler::vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11296 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11297 assert(vector_len <= AVX_256bit, "");
11298 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11299 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11300 int src2_enc = src2->encoding();
11301 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
11302 }
11303
11304 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11305 assert(VM_Version::supports_avx2(), "");
11306 assert(vector_len <= AVX_256bit, "");
11307 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11308 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11309 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
11310 }
11311
11312 void Assembler::vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len) {
11313 assert(VM_Version::supports_avx(), "");
11314 assert(vector_len <= AVX_256bit, "");
11315 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11316 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11317 emit_int24((unsigned char)0xC2, (0xC0 | encode), (unsigned char)comparison);
11318 }
11319
11320 void Assembler::evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11321 ComparisonPredicateFP comparison, int vector_len) {
11322 assert(VM_Version::supports_evex(), "");
11323 // Encoding: EVEX.NDS.XXX.0F.W0 C2 /r ib
11324 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11325 attributes.set_is_evex_instruction();
11326 attributes.set_embedded_opmask_register_specifier(mask);
11327 attributes.reset_is_clear_context();
11328 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
11329 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11330 }
11331
11332 void Assembler::evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11333 ComparisonPredicateFP comparison, int vector_len) {
11334 assert(VM_Version::supports_evex(), "");
11335 // Encoding: EVEX.NDS.XXX.66.0F.W1 C2 /r ib
11336 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11337 attributes.set_is_evex_instruction();
11338 attributes.set_embedded_opmask_register_specifier(mask);
11339 attributes.reset_is_clear_context();
11340 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11341 emit_int24((unsigned char)0xC2, (0xC0 | encode), comparison);
11342 }
11343
11344 void Assembler::blendvps(XMMRegister dst, XMMRegister src) {
11345 assert(VM_Version::supports_sse4_1(), "");
11346 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11347 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11348 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11349 emit_int16(0x14, (0xC0 | encode));
11350 }
11351
11352 void Assembler::blendvpd(XMMRegister dst, XMMRegister src) {
11353 assert(VM_Version::supports_sse4_1(), "");
11354 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11355 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11356 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11357 emit_int16(0x15, (0xC0 | encode));
11358 }
11359
11360 void Assembler::pblendvb(XMMRegister dst, XMMRegister src) {
11361 assert(VM_Version::supports_sse4_1(), "");
11362 assert(UseAVX <= 0, "sse encoding is inconsistent with avx encoding");
11363 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11364 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11365 emit_int16(0x10, (0xC0 | encode));
11366 }
11367
11368 void Assembler::vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
11369 assert(UseAVX > 0 && (vector_len == AVX_128bit || vector_len == AVX_256bit), "");
11370 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11371 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11372 int src2_enc = src2->encoding();
11373 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
11374 }
11375
11376 void Assembler::vblendps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
11377 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11378 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11379 emit_int24(0x0C, (0xC0 | encode), imm8);
11380 }
11381
11382 void Assembler::vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11383 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11384 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11385 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11386 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11387 emit_int16(0x64, (0xC0 | encode));
11388 }
11389
11390 void Assembler::vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11391 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11392 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11393 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11394 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11395 emit_int16(0x65, (0xC0 | encode));
11396 }
11397
11398 void Assembler::vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11399 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11400 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11401 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11402 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11403 emit_int16(0x66, (0xC0 | encode));
11404 }
11405
11406 void Assembler::vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
11407 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() : VM_Version::supports_avx2(), "");
11408 assert(vector_len <= AVX_256bit, "evex encoding is different - has k register as dest");
11409 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11410 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11411 emit_int16(0x37, (0xC0 | encode));
11412 }
11413
11414 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11415 int comparison, bool is_signed, int vector_len) {
11416 assert(VM_Version::supports_evex(), "");
11417 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11418 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11419 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11420 attributes.set_is_evex_instruction();
11421 attributes.set_embedded_opmask_register_specifier(mask);
11422 attributes.reset_is_clear_context();
11423 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11424 int opcode = is_signed ? 0x1F : 0x1E;
11425 emit_int24(opcode, (0xC0 | encode), comparison);
11426 }
11427
11428 void Assembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11429 int comparison, bool is_signed, int vector_len) {
11430 assert(VM_Version::supports_evex(), "");
11431 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11432 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 1F /r ib
11433 InstructionMark im(this);
11434 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11435 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11436 attributes.set_is_evex_instruction();
11437 attributes.set_embedded_opmask_register_specifier(mask);
11438 attributes.reset_is_clear_context();
11439 int dst_enc = kdst->encoding();
11440 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11441 int opcode = is_signed ? 0x1F : 0x1E;
11442 emit_int8((unsigned char)opcode);
11443 emit_operand(as_Register(dst_enc), src);
11444 emit_int8((unsigned char)comparison);
11445 }
11446
11447 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11448 int comparison, bool is_signed, int vector_len) {
11449 assert(VM_Version::supports_evex(), "");
11450 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11451 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11452 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11453 attributes.set_is_evex_instruction();
11454 attributes.set_embedded_opmask_register_specifier(mask);
11455 attributes.reset_is_clear_context();
11456 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11457 int opcode = is_signed ? 0x1F : 0x1E;
11458 emit_int24(opcode, (0xC0 | encode), comparison);
11459 }
11460
11461 void Assembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11462 int comparison, bool is_signed, int vector_len) {
11463 assert(VM_Version::supports_evex(), "");
11464 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11465 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 1F /r ib
11466 InstructionMark im(this);
11467 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11468 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
11469 attributes.set_is_evex_instruction();
11470 attributes.set_embedded_opmask_register_specifier(mask);
11471 attributes.reset_is_clear_context();
11472 int dst_enc = kdst->encoding();
11473 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11474 int opcode = is_signed ? 0x1F : 0x1E;
11475 emit_int8((unsigned char)opcode);
11476 emit_operand(as_Register(dst_enc), src);
11477 emit_int8((unsigned char)comparison);
11478 }
11479
11480 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11481 int comparison, bool is_signed, int vector_len) {
11482 assert(VM_Version::supports_evex(), "");
11483 assert(VM_Version::supports_avx512bw(), "");
11484 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11485 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11486 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11487 attributes.set_is_evex_instruction();
11488 attributes.set_embedded_opmask_register_specifier(mask);
11489 attributes.reset_is_clear_context();
11490 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11491 int opcode = is_signed ? 0x3F : 0x3E;
11492 emit_int24(opcode, (0xC0 | encode), comparison);
11493 }
11494
11495 void Assembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11496 int comparison, bool is_signed, int vector_len) {
11497 assert(VM_Version::supports_evex(), "");
11498 assert(VM_Version::supports_avx512bw(), "");
11499 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11500 // Encoding: EVEX.NDS.XXX.66.0F3A.W0 3F /r ib
11501 InstructionMark im(this);
11502 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11503 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11504 attributes.set_is_evex_instruction();
11505 attributes.set_embedded_opmask_register_specifier(mask);
11506 attributes.reset_is_clear_context();
11507 int dst_enc = kdst->encoding();
11508 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11509 int opcode = is_signed ? 0x3F : 0x3E;
11510 emit_int8((unsigned char)opcode);
11511 emit_operand(as_Register(dst_enc), src);
11512 emit_int8((unsigned char)comparison);
11513 }
11514
11515 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
11516 int comparison, bool is_signed, int vector_len) {
11517 assert(VM_Version::supports_evex(), "");
11518 assert(VM_Version::supports_avx512bw(), "");
11519 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11520 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11521 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11522 attributes.set_is_evex_instruction();
11523 attributes.set_embedded_opmask_register_specifier(mask);
11524 attributes.reset_is_clear_context();
11525 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11526 int opcode = is_signed ? 0x3F : 0x3E;
11527 emit_int24(opcode, (0xC0 | encode), comparison);
11528 }
11529
11530 void Assembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
11531 int comparison, bool is_signed, int vector_len) {
11532 assert(VM_Version::supports_evex(), "");
11533 assert(VM_Version::supports_avx512bw(), "");
11534 assert(comparison >= Assembler::eq && comparison <= Assembler::_true, "");
11535 // Encoding: EVEX.NDS.XXX.66.0F3A.W1 3F /r ib
11536 InstructionMark im(this);
11537 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11538 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
11539 attributes.set_is_evex_instruction();
11540 attributes.set_embedded_opmask_register_specifier(mask);
11541 attributes.reset_is_clear_context();
11542 int dst_enc = kdst->encoding();
11543 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11544 int opcode = is_signed ? 0x3F : 0x3E;
11545 emit_int8((unsigned char)opcode);
11546 emit_operand(as_Register(dst_enc), src);
11547 emit_int8((unsigned char)comparison);
11548 }
11549
11550 // Register is a class, but it would be assigned numerical value.
11551 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
11552 PRAGMA_DIAG_PUSH
11553 PRAGMA_NONNULL_IGNORED
11554 void Assembler::evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11555 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11556 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11557 attributes.set_is_evex_instruction();
11558 attributes.set_embedded_opmask_register_specifier(mask);
11559 if (merge) {
11560 attributes.reset_is_clear_context();
11561 }
11562 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11563 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11564 }
11565
11566 void Assembler::evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11567 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11568 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11569 attributes.set_is_evex_instruction();
11570 attributes.set_embedded_opmask_register_specifier(mask);
11571 if (merge) {
11572 attributes.reset_is_clear_context();
11573 }
11574 int encode = vex_prefix_and_encode(xmm0->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11575 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11576 }
11577 PRAGMA_DIAG_POP
11578
11579 void Assembler::evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11580 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11581 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11582 attributes.set_is_evex_instruction();
11583 attributes.set_embedded_opmask_register_specifier(mask);
11584 if (merge) {
11585 attributes.reset_is_clear_context();
11586 }
11587 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11588 emit_int16(0x14, (0xC0 | encode));
11589 }
11590
11591 void Assembler::evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11592 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11593 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11594 attributes.set_is_evex_instruction();
11595 attributes.set_embedded_opmask_register_specifier(mask);
11596 if (merge) {
11597 attributes.reset_is_clear_context();
11598 }
11599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11600 emit_int16(0x14, (0xC0 | encode));
11601 }
11602
11603 void Assembler::evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11604 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11605 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11606 attributes.set_is_evex_instruction();
11607 attributes.set_embedded_opmask_register_specifier(mask);
11608 if (merge) {
11609 attributes.reset_is_clear_context();
11610 }
11611 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11612 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11613 }
11614
11615 void Assembler::evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len) {
11616 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11617 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11618 attributes.set_is_evex_instruction();
11619 attributes.set_embedded_opmask_register_specifier(mask);
11620 if (merge) {
11621 attributes.reset_is_clear_context();
11622 }
11623 int encode = vex_prefix_and_encode(xmm1->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
11624 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
11625 }
11626
11627 void Assembler::evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11628 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11629 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11630 attributes.set_is_evex_instruction();
11631 attributes.set_embedded_opmask_register_specifier(mask);
11632 if (merge) {
11633 attributes.reset_is_clear_context();
11634 }
11635 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11636 emit_int16(0x15, (0xC0 | encode));
11637 }
11638
11639 void Assembler::evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11640 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11641 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11642 attributes.set_is_evex_instruction();
11643 attributes.set_embedded_opmask_register_specifier(mask);
11644 if (merge) {
11645 attributes.reset_is_clear_context();
11646 }
11647 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11648 emit_int16(0x15, (0xC0 | encode));
11649 }
11650
11651 void Assembler::vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len) {
11652 assert(VM_Version::supports_avx(), "");
11653 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11654 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
11655 int mask_enc = mask->encoding();
11656 emit_int24(0x4C, (0xC0 | encode), 0xF0 & mask_enc << 4);
11657 }
11658
11659 void Assembler::evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11660 assert(VM_Version::supports_evex(), "");
11661 // Encoding: EVEX.NDS.XXX.66.0F38.W1 65 /r
11662 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11663 attributes.set_is_evex_instruction();
11664 attributes.set_embedded_opmask_register_specifier(mask);
11665 if (merge) {
11666 attributes.reset_is_clear_context();
11667 }
11668 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11669 emit_int16(0x65, (0xC0 | encode));
11670 }
11671
11672 void Assembler::evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11673 assert(VM_Version::supports_evex(), "");
11674 // Encoding: EVEX.NDS.XXX.66.0F38.W0 65 /r
11675 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11676 attributes.set_is_evex_instruction();
11677 attributes.set_embedded_opmask_register_specifier(mask);
11678 if (merge) {
11679 attributes.reset_is_clear_context();
11680 }
11681 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11682 emit_int16(0x65, (0xC0 | encode));
11683 }
11684
11685 void Assembler::evpblendmb (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11686 assert(VM_Version::supports_evex(), "");
11687 assert(VM_Version::supports_avx512bw(), "");
11688 // Encoding: EVEX.NDS.512.66.0F38.W0 66 /r
11689 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11690 attributes.set_is_evex_instruction();
11691 attributes.set_embedded_opmask_register_specifier(mask);
11692 if (merge) {
11693 attributes.reset_is_clear_context();
11694 }
11695 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11696 emit_int16(0x66, (0xC0 | encode));
11697 }
11698
11699 void Assembler::evpblendmw (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11700 assert(VM_Version::supports_evex(), "");
11701 assert(VM_Version::supports_avx512bw(), "");
11702 // Encoding: EVEX.NDS.512.66.0F38.W1 66 /r
11703 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
11704 attributes.set_is_evex_instruction();
11705 attributes.set_embedded_opmask_register_specifier(mask);
11706 if (merge) {
11707 attributes.reset_is_clear_context();
11708 }
11709 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11710 emit_int16(0x66, (0xC0 | encode));
11711 }
11712
11713 void Assembler::evpblendmd (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11714 assert(VM_Version::supports_evex(), "");
11715 //Encoding: EVEX.NDS.512.66.0F38.W0 64 /r
11716 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11717 attributes.set_is_evex_instruction();
11718 attributes.set_embedded_opmask_register_specifier(mask);
11719 if (merge) {
11720 attributes.reset_is_clear_context();
11721 }
11722 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11723 emit_int16(0x64, (0xC0 | encode));
11724 }
11725
11726 void Assembler::evpblendmq (XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
11727 assert(VM_Version::supports_evex(), "");
11728 //Encoding: EVEX.NDS.512.66.0F38.W1 64 /r
11729 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11730 attributes.set_is_evex_instruction();
11731 attributes.set_embedded_opmask_register_specifier(mask);
11732 if (merge) {
11733 attributes.reset_is_clear_context();
11734 }
11735 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11736 emit_int16(0x64, (0xC0 | encode));
11737 }
11738
11739 void Assembler::bzhiq(Register dst, Register src1, Register src2) {
11740 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11741 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11742 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
11743 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11744 }
11745
11746 void Assembler::pextl(Register dst, Register src1, Register src2) {
11747 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11748 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11749 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11750 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11751 }
11752
11753 void Assembler::pdepl(Register dst, Register src1, Register src2) {
11754 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11755 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11756 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11757 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11758 }
11759
11760 void Assembler::pextq(Register dst, Register src1, Register src2) {
11761 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11762 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11763 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11764 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11765 }
11766
11767 void Assembler::pdepq(Register dst, Register src1, Register src2) {
11768 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11769 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11770 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11771 emit_int16((unsigned char)0xF5, (0xC0 | encode));
11772 }
11773
11774 void Assembler::pextl(Register dst, Register src1, Address src2) {
11775 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11776 InstructionMark im(this);
11777 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11778 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11779 emit_int8((unsigned char)0xF5);
11780 emit_operand(dst, src2);
11781 }
11782
11783 void Assembler::pdepl(Register dst, Register src1, Address src2) {
11784 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11785 InstructionMark im(this);
11786 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11787 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11788 emit_int8((unsigned char)0xF5);
11789 emit_operand(dst, src2);
11790 }
11791
11792 void Assembler::pextq(Register dst, Register src1, Address src2) {
11793 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11794 InstructionMark im(this);
11795 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11796 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11797 emit_int8((unsigned char)0xF5);
11798 emit_operand(dst, src2);
11799 }
11800
11801 void Assembler::pdepq(Register dst, Register src1, Address src2) {
11802 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
11803 InstructionMark im(this);
11804 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
11805 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11806 emit_int8((unsigned char)0xF5);
11807 emit_operand(dst, src2);
11808 }
11809
11810 void Assembler::sarxl(Register dst, Register src1, Register src2) {
11811 assert(VM_Version::supports_bmi2(), "");
11812 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11813 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11814 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11815 }
11816
11817 void Assembler::sarxl(Register dst, Address src1, Register src2) {
11818 assert(VM_Version::supports_bmi2(), "");
11819 InstructionMark im(this);
11820 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11821 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11822 emit_int8((unsigned char)0xF7);
11823 emit_operand(dst, src1);
11824 }
11825
11826 void Assembler::sarxq(Register dst, Register src1, Register src2) {
11827 assert(VM_Version::supports_bmi2(), "");
11828 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11829 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11830 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11831 }
11832
11833 void Assembler::sarxq(Register dst, Address src1, Register src2) {
11834 assert(VM_Version::supports_bmi2(), "");
11835 InstructionMark im(this);
11836 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11837 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11838 emit_int8((unsigned char)0xF7);
11839 emit_operand(dst, src1);
11840 }
11841
11842 void Assembler::shlxl(Register dst, Register src1, Register src2) {
11843 assert(VM_Version::supports_bmi2(), "");
11844 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11845 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11846 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11847 }
11848
11849 void Assembler::shlxl(Register dst, Address src1, Register src2) {
11850 assert(VM_Version::supports_bmi2(), "");
11851 InstructionMark im(this);
11852 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11853 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11854 emit_int8((unsigned char)0xF7);
11855 emit_operand(dst, src1);
11856 }
11857
11858 void Assembler::shlxq(Register dst, Register src1, Register src2) {
11859 assert(VM_Version::supports_bmi2(), "");
11860 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11861 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11862 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11863 }
11864
11865 void Assembler::shlxq(Register dst, Address src1, Register src2) {
11866 assert(VM_Version::supports_bmi2(), "");
11867 InstructionMark im(this);
11868 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11869 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11870 emit_int8((unsigned char)0xF7);
11871 emit_operand(dst, src1);
11872 }
11873
11874 void Assembler::shrxl(Register dst, Register src1, Register src2) {
11875 assert(VM_Version::supports_bmi2(), "");
11876 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11877 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11878 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11879 }
11880
11881 void Assembler::shrxl(Register dst, Address src1, Register src2) {
11882 assert(VM_Version::supports_bmi2(), "");
11883 InstructionMark im(this);
11884 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11885 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11886 emit_int8((unsigned char)0xF7);
11887 emit_operand(dst, src1);
11888 }
11889
11890 void Assembler::shrxq(Register dst, Register src1, Register src2) {
11891 assert(VM_Version::supports_bmi2(), "");
11892 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11893 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11894 emit_int16((unsigned char)0xF7, (0xC0 | encode));
11895 }
11896
11897 void Assembler::shrxq(Register dst, Address src1, Register src2) {
11898 assert(VM_Version::supports_bmi2(), "");
11899 InstructionMark im(this);
11900 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
11901 vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
11902 emit_int8((unsigned char)0xF7);
11903 emit_operand(dst, src1);
11904 }
11905
11906 void Assembler::evpmovq2m(KRegister dst, XMMRegister src, int vector_len) {
11907 assert(VM_Version::supports_avx512vldq(), "");
11908 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11909 attributes.set_is_evex_instruction();
11910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11911 emit_int16(0x39, (0xC0 | encode));
11912 }
11913
11914 void Assembler::evpmovd2m(KRegister dst, XMMRegister src, int vector_len) {
11915 assert(VM_Version::supports_avx512vldq(), "");
11916 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11917 attributes.set_is_evex_instruction();
11918 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11919 emit_int16(0x39, (0xC0 | encode));
11920 }
11921
11922 void Assembler::evpmovw2m(KRegister dst, XMMRegister src, int vector_len) {
11923 assert(VM_Version::supports_avx512vlbw(), "");
11924 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11925 attributes.set_is_evex_instruction();
11926 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11927 emit_int16(0x29, (0xC0 | encode));
11928 }
11929
11930 void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) {
11931 assert(VM_Version::supports_avx512vlbw(), "");
11932 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11933 attributes.set_is_evex_instruction();
11934 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11935 emit_int16(0x29, (0xC0 | encode));
11936 }
11937
11938 void Assembler::evpmovm2q(XMMRegister dst, KRegister src, int vector_len) {
11939 assert(VM_Version::supports_avx512vldq(), "");
11940 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11941 attributes.set_is_evex_instruction();
11942 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11943 emit_int16(0x38, (0xC0 | encode));
11944 }
11945
11946 void Assembler::evpmovm2d(XMMRegister dst, KRegister src, int vector_len) {
11947 assert(VM_Version::supports_avx512vldq(), "");
11948 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11949 attributes.set_is_evex_instruction();
11950 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11951 emit_int16(0x38, (0xC0 | encode));
11952 }
11953
11954 void Assembler::evpmovm2w(XMMRegister dst, KRegister src, int vector_len) {
11955 assert(VM_Version::supports_avx512vlbw(), "");
11956 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11957 attributes.set_is_evex_instruction();
11958 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11959 emit_int16(0x28, (0xC0 | encode));
11960 }
11961
11962 void Assembler::evpmovm2b(XMMRegister dst, KRegister src, int vector_len) {
11963 assert(VM_Version::supports_avx512vlbw(), "");
11964 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
11965 attributes.set_is_evex_instruction();
11966 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
11967 emit_int16(0x28, (0xC0 | encode));
11968 }
11969
11970 void Assembler::evpcompressb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11971 assert(VM_Version::supports_avx512_vbmi2(), "");
11972 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11973 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11974 attributes.set_embedded_opmask_register_specifier(mask);
11975 attributes.set_is_evex_instruction();
11976 if (merge) {
11977 attributes.reset_is_clear_context();
11978 }
11979 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11980 emit_int16((unsigned char)0x63, (0xC0 | encode));
11981 }
11982
11983 void Assembler::evpcompressw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11984 assert(VM_Version::supports_avx512_vbmi2(), "");
11985 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11986 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
11987 attributes.set_embedded_opmask_register_specifier(mask);
11988 attributes.set_is_evex_instruction();
11989 if (merge) {
11990 attributes.reset_is_clear_context();
11991 }
11992 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
11993 emit_int16((unsigned char)0x63, (0xC0 | encode));
11994 }
11995
11996 void Assembler::evpcompressd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
11997 assert(VM_Version::supports_evex(), "");
11998 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
11999 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12000 attributes.set_embedded_opmask_register_specifier(mask);
12001 attributes.set_is_evex_instruction();
12002 if (merge) {
12003 attributes.reset_is_clear_context();
12004 }
12005 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12006 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12007 }
12008
12009 void Assembler::evpcompressq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12010 assert(VM_Version::supports_evex(), "");
12011 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12012 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12013 attributes.set_embedded_opmask_register_specifier(mask);
12014 attributes.set_is_evex_instruction();
12015 if (merge) {
12016 attributes.reset_is_clear_context();
12017 }
12018 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12019 emit_int16((unsigned char)0x8B, (0xC0 | encode));
12020 }
12021
12022 void Assembler::evcompressps(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12023 assert(VM_Version::supports_evex(), "");
12024 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12025 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12026 attributes.set_embedded_opmask_register_specifier(mask);
12027 attributes.set_is_evex_instruction();
12028 if (merge) {
12029 attributes.reset_is_clear_context();
12030 }
12031 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12032 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12033 }
12034
12035 void Assembler::evcompresspd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
12036 assert(VM_Version::supports_evex(), "");
12037 assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
12038 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
12039 attributes.set_embedded_opmask_register_specifier(mask);
12040 attributes.set_is_evex_instruction();
12041 if (merge) {
12042 attributes.reset_is_clear_context();
12043 }
12044 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
12045 emit_int16((unsigned char)0x8A, (0xC0 | encode));
12046 }
12047
12048 #ifndef _LP64
12049
12050 void Assembler::incl(Register dst) {
12051 // Don't use it directly. Use MacroAssembler::incrementl() instead.
12052 emit_int8(0x40 | dst->encoding());
12053 }
12054
12055 void Assembler::lea(Register dst, Address src) {
12056 leal(dst, src);
12057 }
12058
12059 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12060 InstructionMark im(this);
12061 emit_int8((unsigned char)0xC7);
12062 emit_operand(rax, dst);
12063 emit_data((int)imm32, rspec, 0);
12064 }
12065
12066 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12067 InstructionMark im(this);
12068 int encode = prefix_and_encode(dst->encoding());
12069 emit_int8((0xB8 | encode));
12070 emit_data((int)imm32, rspec, 0);
12071 }
12072
12073 void Assembler::popa() { // 32bit
12074 emit_int8(0x61);
12075 }
12076
12077 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
12078 InstructionMark im(this);
12079 emit_int8(0x68);
12080 emit_data(imm32, rspec, 0);
12081 }
12082
12083 void Assembler::pusha() { // 32bit
12084 emit_int8(0x60);
12085 }
12086
12087 void Assembler::set_byte_if_not_zero(Register dst) {
12088 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | dst->encoding()));
12089 }
12090
12091 #else // LP64
12092
12093 void Assembler::set_byte_if_not_zero(Register dst) {
12094 int enc = prefix_and_encode(dst->encoding(), true);
12095 emit_int24(0x0F, (unsigned char)0x95, (0xC0 | enc));
12096 }
12097
12098 // 64bit only pieces of the assembler
12099 // This should only be used by 64bit instructions that can use rip-relative
12100 // it cannot be used by instructions that want an immediate value.
12101
12102 bool Assembler::reachable(AddressLiteral adr) {
12103 int64_t disp;
12104 relocInfo::relocType relocType = adr.reloc();
12105
12106 // None will force a 64bit literal to the code stream. Likely a placeholder
12107 // for something that will be patched later and we need to certain it will
12108 // always be reachable.
12109 if (relocType == relocInfo::none) {
12110 return false;
12111 }
12112 if (relocType == relocInfo::internal_word_type) {
12113 // This should be rip relative and easily reachable.
12114 return true;
12115 }
12116 if (relocType == relocInfo::virtual_call_type ||
12117 relocType == relocInfo::opt_virtual_call_type ||
12118 relocType == relocInfo::static_call_type ||
12119 relocType == relocInfo::static_stub_type ) {
12120 // This should be rip relative within the code cache and easily
12121 // reachable until we get huge code caches. (At which point
12122 // ic code is going to have issues).
12123 return true;
12124 }
12125 if (relocType != relocInfo::external_word_type &&
12126 relocType != relocInfo::poll_return_type && // these are really external_word but need special
12127 relocType != relocInfo::poll_type && // relocs to identify them
12128 relocType != relocInfo::runtime_call_type ) {
12129 return false;
12130 }
12131
12132 // Stress the correction code
12133 if (ForceUnreachable) {
12134 // Must be runtimecall reloc, see if it is in the codecache
12135 // Flipping stuff in the codecache to be unreachable causes issues
12136 // with things like inline caches where the additional instructions
12137 // are not handled.
12138 if (CodeCache::find_blob(adr._target) == NULL) {
12139 return false;
12140 }
12141 }
12142 // For external_word_type/runtime_call_type if it is reachable from where we
12143 // are now (possibly a temp buffer) and where we might end up
12144 // anywhere in the codeCache then we are always reachable.
12145 // This would have to change if we ever save/restore shared code
12146 // to be more pessimistic.
12147 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
12148 if (!is_simm32(disp)) return false;
12149 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
12150 if (!is_simm32(disp)) return false;
12151
12152 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
12153
12154 // Because rip relative is a disp + address_of_next_instruction and we
12155 // don't know the value of address_of_next_instruction we apply a fudge factor
12156 // to make sure we will be ok no matter the size of the instruction we get placed into.
12157 // We don't have to fudge the checks above here because they are already worst case.
12158
12159 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
12160 // + 4 because better safe than sorry.
12161 const int fudge = 12 + 4;
12162 if (disp < 0) {
12163 disp -= fudge;
12164 } else {
12165 disp += fudge;
12166 }
12167 return is_simm32(disp);
12168 }
12169
12170 void Assembler::emit_data64(jlong data,
12171 relocInfo::relocType rtype,
12172 int format) {
12173 if (rtype == relocInfo::none) {
12174 emit_int64(data);
12175 } else {
12176 emit_data64(data, Relocation::spec_simple(rtype), format);
12177 }
12178 }
12179
12180 void Assembler::emit_data64(jlong data,
12181 RelocationHolder const& rspec,
12182 int format) {
12183 assert(imm_operand == 0, "default format must be immediate in this file");
12184 assert(imm_operand == format, "must be immediate");
12185 assert(inst_mark() != NULL, "must be inside InstructionMark");
12186 // Do not use AbstractAssembler::relocate, which is not intended for
12187 // embedded words. Instead, relocate to the enclosing instruction.
12188 code_section()->relocate(inst_mark(), rspec, format);
12189 #ifdef ASSERT
12190 check_relocation(rspec, format);
12191 #endif
12192 emit_int64(data);
12193 }
12194
12195 void Assembler::prefix(Register reg) {
12196 if (reg->encoding() >= 8) {
12197 prefix(REX_B);
12198 }
12199 }
12200
12201 void Assembler::prefix(Register dst, Register src, Prefix p) {
12202 if (src->encoding() >= 8) {
12203 p = (Prefix)(p | REX_B);
12204 }
12205 if (dst->encoding() >= 8) {
12206 p = (Prefix)(p | REX_R);
12207 }
12208 if (p != Prefix_EMPTY) {
12209 // do not generate an empty prefix
12210 prefix(p);
12211 }
12212 }
12213
12214 void Assembler::prefix(Register dst, Address adr, Prefix p) {
12215 if (adr.base_needs_rex()) {
12216 if (adr.index_needs_rex()) {
12217 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12218 } else {
12219 prefix(REX_B);
12220 }
12221 } else {
12222 if (adr.index_needs_rex()) {
12223 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
12224 }
12225 }
12226 if (dst->encoding() >= 8) {
12227 p = (Prefix)(p | REX_R);
12228 }
12229 if (p != Prefix_EMPTY) {
12230 // do not generate an empty prefix
12231 prefix(p);
12232 }
12233 }
12234
12235 void Assembler::prefix(Address adr) {
12236 if (adr.base_needs_rex()) {
12237 if (adr.index_needs_rex()) {
12238 prefix(REX_XB);
12239 } else {
12240 prefix(REX_B);
12241 }
12242 } else {
12243 if (adr.index_needs_rex()) {
12244 prefix(REX_X);
12245 }
12246 }
12247 }
12248
12249 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
12250 if (reg->encoding() < 8) {
12251 if (adr.base_needs_rex()) {
12252 if (adr.index_needs_rex()) {
12253 prefix(REX_XB);
12254 } else {
12255 prefix(REX_B);
12256 }
12257 } else {
12258 if (adr.index_needs_rex()) {
12259 prefix(REX_X);
12260 } else if (byteinst && reg->encoding() >= 4) {
12261 prefix(REX);
12262 }
12263 }
12264 } else {
12265 if (adr.base_needs_rex()) {
12266 if (adr.index_needs_rex()) {
12267 prefix(REX_RXB);
12268 } else {
12269 prefix(REX_RB);
12270 }
12271 } else {
12272 if (adr.index_needs_rex()) {
12273 prefix(REX_RX);
12274 } else {
12275 prefix(REX_R);
12276 }
12277 }
12278 }
12279 }
12280
12281 void Assembler::prefix(Address adr, XMMRegister reg) {
12282 if (reg->encoding() < 8) {
12283 if (adr.base_needs_rex()) {
12284 if (adr.index_needs_rex()) {
12285 prefix(REX_XB);
12286 } else {
12287 prefix(REX_B);
12288 }
12289 } else {
12290 if (adr.index_needs_rex()) {
12291 prefix(REX_X);
12292 }
12293 }
12294 } else {
12295 if (adr.base_needs_rex()) {
12296 if (adr.index_needs_rex()) {
12297 prefix(REX_RXB);
12298 } else {
12299 prefix(REX_RB);
12300 }
12301 } else {
12302 if (adr.index_needs_rex()) {
12303 prefix(REX_RX);
12304 } else {
12305 prefix(REX_R);
12306 }
12307 }
12308 }
12309 }
12310
12311 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
12312 if (reg_enc >= 8) {
12313 prefix(REX_B);
12314 reg_enc -= 8;
12315 } else if (byteinst && reg_enc >= 4) {
12316 prefix(REX);
12317 }
12318 return reg_enc;
12319 }
12320
12321 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
12322 if (dst_enc < 8) {
12323 if (src_enc >= 8) {
12324 prefix(REX_B);
12325 src_enc -= 8;
12326 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
12327 prefix(REX);
12328 }
12329 } else {
12330 if (src_enc < 8) {
12331 prefix(REX_R);
12332 } else {
12333 prefix(REX_RB);
12334 src_enc -= 8;
12335 }
12336 dst_enc -= 8;
12337 }
12338 return dst_enc << 3 | src_enc;
12339 }
12340
12341 int8_t Assembler::get_prefixq(Address adr) {
12342 int8_t prfx = get_prefixq(adr, rax);
12343 assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
12344 return prfx;
12345 }
12346
12347 int8_t Assembler::get_prefixq(Address adr, Register src) {
12348 int8_t prfx = (int8_t)(REX_W +
12349 ((int)adr.base_needs_rex()) +
12350 ((int)adr.index_needs_rex() << 1) +
12351 ((int)(src->encoding() >= 8) << 2));
12352 #ifdef ASSERT
12353 if (src->encoding() < 8) {
12354 if (adr.base_needs_rex()) {
12355 if (adr.index_needs_rex()) {
12356 assert(prfx == REX_WXB, "must be");
12357 } else {
12358 assert(prfx == REX_WB, "must be");
12359 }
12360 } else {
12361 if (adr.index_needs_rex()) {
12362 assert(prfx == REX_WX, "must be");
12363 } else {
12364 assert(prfx == REX_W, "must be");
12365 }
12366 }
12367 } else {
12368 if (adr.base_needs_rex()) {
12369 if (adr.index_needs_rex()) {
12370 assert(prfx == REX_WRXB, "must be");
12371 } else {
12372 assert(prfx == REX_WRB, "must be");
12373 }
12374 } else {
12375 if (adr.index_needs_rex()) {
12376 assert(prfx == REX_WRX, "must be");
12377 } else {
12378 assert(prfx == REX_WR, "must be");
12379 }
12380 }
12381 }
12382 #endif
12383 return prfx;
12384 }
12385
12386 void Assembler::prefixq(Address adr) {
12387 emit_int8(get_prefixq(adr));
12388 }
12389
12390 void Assembler::prefixq(Address adr, Register src) {
12391 emit_int8(get_prefixq(adr, src));
12392 }
12393
12394 void Assembler::prefixq(Address adr, XMMRegister src) {
12395 if (src->encoding() < 8) {
12396 if (adr.base_needs_rex()) {
12397 if (adr.index_needs_rex()) {
12398 prefix(REX_WXB);
12399 } else {
12400 prefix(REX_WB);
12401 }
12402 } else {
12403 if (adr.index_needs_rex()) {
12404 prefix(REX_WX);
12405 } else {
12406 prefix(REX_W);
12407 }
12408 }
12409 } else {
12410 if (adr.base_needs_rex()) {
12411 if (adr.index_needs_rex()) {
12412 prefix(REX_WRXB);
12413 } else {
12414 prefix(REX_WRB);
12415 }
12416 } else {
12417 if (adr.index_needs_rex()) {
12418 prefix(REX_WRX);
12419 } else {
12420 prefix(REX_WR);
12421 }
12422 }
12423 }
12424 }
12425
12426 int Assembler::prefixq_and_encode(int reg_enc) {
12427 if (reg_enc < 8) {
12428 prefix(REX_W);
12429 } else {
12430 prefix(REX_WB);
12431 reg_enc -= 8;
12432 }
12433 return reg_enc;
12434 }
12435
12436 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
12437 if (dst_enc < 8) {
12438 if (src_enc < 8) {
12439 prefix(REX_W);
12440 } else {
12441 prefix(REX_WB);
12442 src_enc -= 8;
12443 }
12444 } else {
12445 if (src_enc < 8) {
12446 prefix(REX_WR);
12447 } else {
12448 prefix(REX_WRB);
12449 src_enc -= 8;
12450 }
12451 dst_enc -= 8;
12452 }
12453 return dst_enc << 3 | src_enc;
12454 }
12455
12456 void Assembler::adcq(Register dst, int32_t imm32) {
12457 (void) prefixq_and_encode(dst->encoding());
12458 emit_arith(0x81, 0xD0, dst, imm32);
12459 }
12460
12461 void Assembler::adcq(Register dst, Address src) {
12462 InstructionMark im(this);
12463 emit_int16(get_prefixq(src, dst), 0x13);
12464 emit_operand(dst, src);
12465 }
12466
12467 void Assembler::adcq(Register dst, Register src) {
12468 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12469 emit_arith(0x13, 0xC0, dst, src);
12470 }
12471
12472 void Assembler::addq(Address dst, int32_t imm32) {
12473 InstructionMark im(this);
12474 prefixq(dst);
12475 emit_arith_operand(0x81, rax, dst, imm32);
12476 }
12477
12478 void Assembler::addq(Address dst, Register src) {
12479 InstructionMark im(this);
12480 emit_int16(get_prefixq(dst, src), 0x01);
12481 emit_operand(src, dst);
12482 }
12483
12484 void Assembler::addq(Register dst, int32_t imm32) {
12485 (void) prefixq_and_encode(dst->encoding());
12486 emit_arith(0x81, 0xC0, dst, imm32);
12487 }
12488
12489 void Assembler::addq(Register dst, Address src) {
12490 InstructionMark im(this);
12491 emit_int16(get_prefixq(src, dst), 0x03);
12492 emit_operand(dst, src);
12493 }
12494
12495 void Assembler::addq(Register dst, Register src) {
12496 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12497 emit_arith(0x03, 0xC0, dst, src);
12498 }
12499
12500 void Assembler::adcxq(Register dst, Register src) {
12501 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12502 emit_int8(0x66);
12503 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12504 emit_int32(0x0F,
12505 0x38,
12506 (unsigned char)0xF6,
12507 (0xC0 | encode));
12508 }
12509
12510 void Assembler::adoxq(Register dst, Register src) {
12511 //assert(VM_Version::supports_adx(), "adx instructions not supported");
12512 emit_int8((unsigned char)0xF3);
12513 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12514 emit_int32(0x0F,
12515 0x38,
12516 (unsigned char)0xF6,
12517 (0xC0 | encode));
12518 }
12519
12520 void Assembler::andq(Address dst, int32_t imm32) {
12521 InstructionMark im(this);
12522 prefixq(dst);
12523 emit_arith_operand(0x81, as_Register(4), dst, imm32);
12524 }
12525
12526 void Assembler::andq(Register dst, int32_t imm32) {
12527 (void) prefixq_and_encode(dst->encoding());
12528 emit_arith(0x81, 0xE0, dst, imm32);
12529 }
12530
12531 void Assembler::andq(Register dst, Address src) {
12532 InstructionMark im(this);
12533 emit_int16(get_prefixq(src, dst), 0x23);
12534 emit_operand(dst, src);
12535 }
12536
12537 void Assembler::andq(Register dst, Register src) {
12538 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12539 emit_arith(0x23, 0xC0, dst, src);
12540 }
12541
12542 void Assembler::andq(Address dst, Register src) {
12543 InstructionMark im(this);
12544 emit_int16(get_prefixq(dst, src), 0x21);
12545 emit_operand(src, dst);
12546 }
12547
12548 void Assembler::andnq(Register dst, Register src1, Register src2) {
12549 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12550 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12551 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12552 emit_int16((unsigned char)0xF2, (0xC0 | encode));
12553 }
12554
12555 void Assembler::andnq(Register dst, Register src1, Address src2) {
12556 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12557 InstructionMark im(this);
12558 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12559 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12560 emit_int8((unsigned char)0xF2);
12561 emit_operand(dst, src2);
12562 }
12563
12564 void Assembler::bsfq(Register dst, Register src) {
12565 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12566 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
12567 }
12568
12569 void Assembler::bsrq(Register dst, Register src) {
12570 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12571 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12572 }
12573
12574 void Assembler::bswapq(Register reg) {
12575 int encode = prefixq_and_encode(reg->encoding());
12576 emit_int16(0x0F, (0xC8 | encode));
12577 }
12578
12579 void Assembler::blsiq(Register dst, Register src) {
12580 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12581 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12582 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12583 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12584 }
12585
12586 void Assembler::blsiq(Register dst, Address src) {
12587 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12588 InstructionMark im(this);
12589 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12590 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12591 emit_int8((unsigned char)0xF3);
12592 emit_operand(rbx, src);
12593 }
12594
12595 void Assembler::blsmskq(Register dst, Register src) {
12596 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12597 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12598 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12599 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12600 }
12601
12602 void Assembler::blsmskq(Register dst, Address src) {
12603 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12604 InstructionMark im(this);
12605 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12606 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12607 emit_int8((unsigned char)0xF3);
12608 emit_operand(rdx, src);
12609 }
12610
12611 void Assembler::blsrq(Register dst, Register src) {
12612 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12613 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12614 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12615 emit_int16((unsigned char)0xF3, (0xC0 | encode));
12616 }
12617
12618 void Assembler::blsrq(Register dst, Address src) {
12619 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
12620 InstructionMark im(this);
12621 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
12622 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
12623 emit_int8((unsigned char)0xF3);
12624 emit_operand(rcx, src);
12625 }
12626
12627 void Assembler::cdqq() {
12628 emit_int16(REX_W, (unsigned char)0x99);
12629 }
12630
12631 void Assembler::clflush(Address adr) {
12632 assert(VM_Version::supports_clflush(), "should do");
12633 prefix(adr);
12634 emit_int16(0x0F, (unsigned char)0xAE);
12635 emit_operand(rdi, adr);
12636 }
12637
12638 void Assembler::clflushopt(Address adr) {
12639 assert(VM_Version::supports_clflushopt(), "should do!");
12640 // adr should be base reg only with no index or offset
12641 assert(adr.index() == noreg, "index should be noreg");
12642 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12643 assert(adr.disp() == 0, "displacement should be 0");
12644 // instruction prefix is 0x66
12645 emit_int8(0x66);
12646 prefix(adr);
12647 // opcode family is 0x0F 0xAE
12648 emit_int16(0x0F, (unsigned char)0xAE);
12649 // extended opcode byte is 7 == rdi
12650 emit_operand(rdi, adr);
12651 }
12652
12653 void Assembler::clwb(Address adr) {
12654 assert(VM_Version::supports_clwb(), "should do!");
12655 // adr should be base reg only with no index or offset
12656 assert(adr.index() == noreg, "index should be noreg");
12657 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
12658 assert(adr.disp() == 0, "displacement should be 0");
12659 // instruction prefix is 0x66
12660 emit_int8(0x66);
12661 prefix(adr);
12662 // opcode family is 0x0f 0xAE
12663 emit_int16(0x0F, (unsigned char)0xAE);
12664 // extended opcode byte is 6 == rsi
12665 emit_operand(rsi, adr);
12666 }
12667
12668 void Assembler::cmovq(Condition cc, Register dst, Register src) {
12669 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12670 emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
12671 }
12672
12673 void Assembler::cmovq(Condition cc, Register dst, Address src) {
12674 InstructionMark im(this);
12675 emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
12676 emit_operand(dst, src);
12677 }
12678
12679 void Assembler::cmpq(Address dst, int32_t imm32) {
12680 InstructionMark im(this);
12681 emit_int16(get_prefixq(dst), (unsigned char)0x81);
12682 emit_operand(rdi, dst, 4);
12683 emit_int32(imm32);
12684 }
12685
12686 void Assembler::cmpq(Register dst, int32_t imm32) {
12687 (void) prefixq_and_encode(dst->encoding());
12688 emit_arith(0x81, 0xF8, dst, imm32);
12689 }
12690
12691 void Assembler::cmpq(Address dst, Register src) {
12692 InstructionMark im(this);
12693 emit_int16(get_prefixq(dst, src), 0x39);
12694 emit_operand(src, dst);
12695 }
12696
12697 void Assembler::cmpq(Register dst, Register src) {
12698 (void) prefixq_and_encode(dst->encoding(), src->encoding());
12699 emit_arith(0x3B, 0xC0, dst, src);
12700 }
12701
12702 void Assembler::cmpq(Register dst, Address src) {
12703 InstructionMark im(this);
12704 emit_int16(get_prefixq(src, dst), 0x3B);
12705 emit_operand(dst, src);
12706 }
12707
12708 void Assembler::cmpxchgq(Register reg, Address adr) {
12709 InstructionMark im(this);
12710 emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
12711 emit_operand(reg, adr);
12712 }
12713
12714 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
12715 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12716 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12717 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12718 emit_int16(0x2A, (0xC0 | encode));
12719 }
12720
12721 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
12722 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12723 InstructionMark im(this);
12724 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12725 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12726 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12727 emit_int8(0x2A);
12728 emit_operand(dst, src);
12729 }
12730
12731 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
12732 NOT_LP64(assert(VM_Version::supports_sse(), ""));
12733 InstructionMark im(this);
12734 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12735 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
12736 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12737 emit_int8(0x2A);
12738 emit_operand(dst, src);
12739 }
12740
12741 void Assembler::cvttsd2siq(Register dst, Address src) {
12742 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12743 // F2 REX.W 0F 2C /r
12744 // CVTTSD2SI r64, xmm1/m64
12745 InstructionMark im(this);
12746 emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
12747 emit_operand(dst, src);
12748 }
12749
12750 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
12751 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12752 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12753 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12754 emit_int16(0x2C, (0xC0 | encode));
12755 }
12756
12757 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
12758 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12759 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12760 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
12761 emit_int16(0x2D, (0xC0 | encode));
12762 }
12763
12764 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
12765 NOT_LP64(assert(VM_Version::supports_sse(), ""));
12766 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12767 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
12768 emit_int16(0x2C, (0xC0 | encode));
12769 }
12770
12771 void Assembler::decl(Register dst) {
12772 // Don't use it directly. Use MacroAssembler::decrementl() instead.
12773 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
12774 int encode = prefix_and_encode(dst->encoding());
12775 emit_int16((unsigned char)0xFF, (0xC8 | encode));
12776 }
12777
12778 void Assembler::decq(Register dst) {
12779 // Don't use it directly. Use MacroAssembler::decrementq() instead.
12780 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12781 int encode = prefixq_and_encode(dst->encoding());
12782 emit_int16((unsigned char)0xFF, 0xC8 | encode);
12783 }
12784
12785 void Assembler::decq(Address dst) {
12786 // Don't use it directly. Use MacroAssembler::decrementq() instead.
12787 InstructionMark im(this);
12788 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12789 emit_operand(rcx, dst);
12790 }
12791
12792 void Assembler::fxrstor(Address src) {
12793 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12794 emit_operand(as_Register(1), src);
12795 }
12796
12797 void Assembler::xrstor(Address src) {
12798 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
12799 emit_operand(as_Register(5), src);
12800 }
12801
12802 void Assembler::fxsave(Address dst) {
12803 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12804 emit_operand(as_Register(0), dst);
12805 }
12806
12807 void Assembler::xsave(Address dst) {
12808 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
12809 emit_operand(as_Register(4), dst);
12810 }
12811
12812 void Assembler::idivq(Register src) {
12813 int encode = prefixq_and_encode(src->encoding());
12814 emit_int16((unsigned char)0xF7, (0xF8 | encode));
12815 }
12816
12817 void Assembler::divq(Register src) {
12818 int encode = prefixq_and_encode(src->encoding());
12819 emit_int16((unsigned char)0xF7, (0xF0 | encode));
12820 }
12821
12822 void Assembler::imulq(Register dst, Register src) {
12823 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12824 emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
12825 }
12826
12827 void Assembler::imulq(Register src) {
12828 int encode = prefixq_and_encode(src->encoding());
12829 emit_int16((unsigned char)0xF7, (0xE8 | encode));
12830 }
12831
12832 void Assembler::imulq(Register dst, Address src, int32_t value) {
12833 InstructionMark im(this);
12834 prefixq(src, dst);
12835 if (is8bit(value)) {
12836 emit_int8((unsigned char)0x6B);
12837 emit_operand(dst, src);
12838 emit_int8(value);
12839 } else {
12840 emit_int8((unsigned char)0x69);
12841 emit_operand(dst, src);
12842 emit_int32(value);
12843 }
12844 }
12845
12846 void Assembler::imulq(Register dst, Register src, int value) {
12847 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12848 if (is8bit(value)) {
12849 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
12850 } else {
12851 emit_int16(0x69, (0xC0 | encode));
12852 emit_int32(value);
12853 }
12854 }
12855
12856 void Assembler::imulq(Register dst, Address src) {
12857 InstructionMark im(this);
12858 emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
12859 emit_operand(dst, src);
12860 }
12861
12862 void Assembler::incl(Register dst) {
12863 // Don't use it directly. Use MacroAssembler::incrementl() instead.
12864 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12865 int encode = prefix_and_encode(dst->encoding());
12866 emit_int16((unsigned char)0xFF, (0xC0 | encode));
12867 }
12868
12869 void Assembler::incq(Register dst) {
12870 // Don't use it directly. Use MacroAssembler::incrementq() instead.
12871 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
12872 int encode = prefixq_and_encode(dst->encoding());
12873 emit_int16((unsigned char)0xFF, (0xC0 | encode));
12874 }
12875
12876 void Assembler::incq(Address dst) {
12877 // Don't use it directly. Use MacroAssembler::incrementq() instead.
12878 InstructionMark im(this);
12879 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
12880 emit_operand(rax, dst);
12881 }
12882
12883 void Assembler::lea(Register dst, Address src) {
12884 leaq(dst, src);
12885 }
12886
12887 void Assembler::leaq(Register dst, Address src) {
12888 InstructionMark im(this);
12889 emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
12890 emit_operand(dst, src);
12891 }
12892
12893 void Assembler::mov64(Register dst, int64_t imm64) {
12894 InstructionMark im(this);
12895 int encode = prefixq_and_encode(dst->encoding());
12896 emit_int8(0xB8 | encode);
12897 emit_int64(imm64);
12898 }
12899
12900 void Assembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
12901 InstructionMark im(this);
12902 int encode = prefixq_and_encode(dst->encoding());
12903 emit_int8(0xB8 | encode);
12904 emit_data64(imm64, rtype, format);
12905 }
12906
12907 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
12908 InstructionMark im(this);
12909 int encode = prefixq_and_encode(dst->encoding());
12910 emit_int8(0xB8 | encode);
12911 emit_data64(imm64, rspec);
12912 }
12913
12914 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
12915 InstructionMark im(this);
12916 int encode = prefix_and_encode(dst->encoding());
12917 emit_int8(0xB8 | encode);
12918 emit_data((int)imm32, rspec, narrow_oop_operand);
12919 }
12920
12921 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
12922 InstructionMark im(this);
12923 prefix(dst);
12924 emit_int8((unsigned char)0xC7);
12925 emit_operand(rax, dst, 4);
12926 emit_data((int)imm32, rspec, narrow_oop_operand);
12927 }
12928
12929 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
12930 InstructionMark im(this);
12931 int encode = prefix_and_encode(src1->encoding());
12932 emit_int16((unsigned char)0x81, (0xF8 | encode));
12933 emit_data((int)imm32, rspec, narrow_oop_operand);
12934 }
12935
12936 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
12937 InstructionMark im(this);
12938 prefix(src1);
12939 emit_int8((unsigned char)0x81);
12940 emit_operand(rax, src1, 4);
12941 emit_data((int)imm32, rspec, narrow_oop_operand);
12942 }
12943
12944 void Assembler::lzcntq(Register dst, Register src) {
12945 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12946 emit_int8((unsigned char)0xF3);
12947 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12948 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
12949 }
12950
12951 void Assembler::lzcntq(Register dst, Address src) {
12952 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
12953 InstructionMark im(this);
12954 emit_int8((unsigned char)0xF3);
12955 prefixq(src, dst);
12956 emit_int16(0x0F, (unsigned char)0xBD);
12957 emit_operand(dst, src);
12958 }
12959
12960 void Assembler::movdq(XMMRegister dst, Register src) {
12961 // table D-1 says MMX/SSE2
12962 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12963 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12964 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12965 emit_int16(0x6E, (0xC0 | encode));
12966 }
12967
12968 void Assembler::movdq(Register dst, XMMRegister src) {
12969 // table D-1 says MMX/SSE2
12970 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
12971 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
12972 // swap src/dst to get correct prefix
12973 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
12974 emit_int16(0x7E,
12975 (0xC0 | encode));
12976 }
12977
12978 void Assembler::movq(Register dst, Register src) {
12979 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
12980 emit_int16((unsigned char)0x8B,
12981 (0xC0 | encode));
12982 }
12983
12984 void Assembler::movq(Register dst, Address src) {
12985 InstructionMark im(this);
12986 emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
12987 emit_operand(dst, src);
12988 }
12989
12990 void Assembler::movq(Address dst, Register src) {
12991 InstructionMark im(this);
12992 emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
12993 emit_operand(src, dst);
12994 }
12995
12996 void Assembler::movq(Address dst, int32_t imm32) {
12997 InstructionMark im(this);
12998 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
12999 emit_operand(as_Register(0), dst);
13000 emit_int32(imm32);
13001 }
13002
13003 void Assembler::movq(Register dst, int32_t imm32) {
13004 int encode = prefixq_and_encode(dst->encoding());
13005 emit_int16((unsigned char)0xC7, (0xC0 | encode));
13006 emit_int32(imm32);
13007 }
13008
13009 void Assembler::movsbq(Register dst, Address src) {
13010 InstructionMark im(this);
13011 emit_int24(get_prefixq(src, dst),
13012 0x0F,
13013 (unsigned char)0xBE);
13014 emit_operand(dst, src);
13015 }
13016
13017 void Assembler::movsbq(Register dst, Register src) {
13018 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13019 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
13020 }
13021
13022 void Assembler::movslq(Register dst, int32_t imm32) {
13023 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
13024 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
13025 // as a result we shouldn't use until tested at runtime...
13026 ShouldNotReachHere();
13027 InstructionMark im(this);
13028 int encode = prefixq_and_encode(dst->encoding());
13029 emit_int8(0xC7 | encode);
13030 emit_int32(imm32);
13031 }
13032
13033 void Assembler::movslq(Address dst, int32_t imm32) {
13034 assert(is_simm32(imm32), "lost bits");
13035 InstructionMark im(this);
13036 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
13037 emit_operand(rax, dst, 4);
13038 emit_int32(imm32);
13039 }
13040
13041 void Assembler::movslq(Register dst, Address src) {
13042 InstructionMark im(this);
13043 emit_int16(get_prefixq(src, dst), 0x63);
13044 emit_operand(dst, src);
13045 }
13046
13047 void Assembler::movslq(Register dst, Register src) {
13048 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13049 emit_int16(0x63, (0xC0 | encode));
13050 }
13051
13052 void Assembler::movswq(Register dst, Address src) {
13053 InstructionMark im(this);
13054 emit_int24(get_prefixq(src, dst),
13055 0x0F,
13056 (unsigned char)0xBF);
13057 emit_operand(dst, src);
13058 }
13059
13060 void Assembler::movswq(Register dst, Register src) {
13061 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13062 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
13063 }
13064
13065 void Assembler::movzbq(Register dst, Address src) {
13066 InstructionMark im(this);
13067 emit_int24(get_prefixq(src, dst),
13068 0x0F,
13069 (unsigned char)0xB6);
13070 emit_operand(dst, src);
13071 }
13072
13073 void Assembler::movzbq(Register dst, Register src) {
13074 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13075 emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
13076 }
13077
13078 void Assembler::movzwq(Register dst, Address src) {
13079 InstructionMark im(this);
13080 emit_int24(get_prefixq(src, dst),
13081 0x0F,
13082 (unsigned char)0xB7);
13083 emit_operand(dst, src);
13084 }
13085
13086 void Assembler::movzwq(Register dst, Register src) {
13087 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13088 emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
13089 }
13090
13091 void Assembler::mulq(Address src) {
13092 InstructionMark im(this);
13093 emit_int16(get_prefixq(src), (unsigned char)0xF7);
13094 emit_operand(rsp, src);
13095 }
13096
13097 void Assembler::mulq(Register src) {
13098 int encode = prefixq_and_encode(src->encoding());
13099 emit_int16((unsigned char)0xF7, (0xE0 | encode));
13100 }
13101
13102 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
13103 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13104 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13105 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
13106 emit_int16((unsigned char)0xF6, (0xC0 | encode));
13107 }
13108
13109 void Assembler::negq(Register dst) {
13110 int encode = prefixq_and_encode(dst->encoding());
13111 emit_int16((unsigned char)0xF7, (0xD8 | encode));
13112 }
13113
13114 void Assembler::negq(Address dst) {
13115 InstructionMark im(this);
13116 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13117 emit_operand(as_Register(3), dst);
13118 }
13119
13120 void Assembler::notq(Register dst) {
13121 int encode = prefixq_and_encode(dst->encoding());
13122 emit_int16((unsigned char)0xF7, (0xD0 | encode));
13123 }
13124
13125 void Assembler::btsq(Address dst, int imm8) {
13126 assert(isByte(imm8), "not a byte");
13127 InstructionMark im(this);
13128 emit_int24(get_prefixq(dst),
13129 0x0F,
13130 (unsigned char)0xBA);
13131 emit_operand(rbp /* 5 */, dst, 1);
13132 emit_int8(imm8);
13133 }
13134
13135 void Assembler::btrq(Address dst, int imm8) {
13136 assert(isByte(imm8), "not a byte");
13137 InstructionMark im(this);
13138 emit_int24(get_prefixq(dst),
13139 0x0F,
13140 (unsigned char)0xBA);
13141 emit_operand(rsi /* 6 */, dst, 1);
13142 emit_int8(imm8);
13143 }
13144
13145 void Assembler::orq(Address dst, int32_t imm32) {
13146 InstructionMark im(this);
13147 prefixq(dst);
13148 emit_arith_operand(0x81, as_Register(1), dst, imm32);
13149 }
13150
13151 void Assembler::orq(Address dst, Register src) {
13152 InstructionMark im(this);
13153 emit_int16(get_prefixq(dst, src), (unsigned char)0x09);
13154 emit_operand(src, dst);
13155 }
13156
13157 void Assembler::orq(Register dst, int32_t imm32) {
13158 (void) prefixq_and_encode(dst->encoding());
13159 emit_arith(0x81, 0xC8, dst, imm32);
13160 }
13161
13162 void Assembler::orq(Register dst, Address src) {
13163 InstructionMark im(this);
13164 emit_int16(get_prefixq(src, dst), 0x0B);
13165 emit_operand(dst, src);
13166 }
13167
13168 void Assembler::orq(Register dst, Register src) {
13169 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13170 emit_arith(0x0B, 0xC0, dst, src);
13171 }
13172
13173 void Assembler::popcntq(Register dst, Address src) {
13174 assert(VM_Version::supports_popcnt(), "must support");
13175 InstructionMark im(this);
13176 emit_int32((unsigned char)0xF3,
13177 get_prefixq(src, dst),
13178 0x0F,
13179 (unsigned char)0xB8);
13180 emit_operand(dst, src);
13181 }
13182
13183 void Assembler::popcntq(Register dst, Register src) {
13184 assert(VM_Version::supports_popcnt(), "must support");
13185 emit_int8((unsigned char)0xF3);
13186 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13187 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
13188 }
13189
13190 void Assembler::popq(Address dst) {
13191 InstructionMark im(this);
13192 emit_int16(get_prefixq(dst), (unsigned char)0x8F);
13193 emit_operand(rax, dst);
13194 }
13195
13196 void Assembler::popq(Register dst) {
13197 emit_int8((unsigned char)0x58 | dst->encoding());
13198 }
13199
13200 // Precomputable: popa, pusha, vzeroupper
13201
13202 // The result of these routines are invariant from one invocation to another
13203 // invocation for the duration of a run. Caching the result on bootstrap
13204 // and copying it out on subsequent invocations can thus be beneficial
13205 static bool precomputed = false;
13206
13207 static u_char* popa_code = NULL;
13208 static int popa_len = 0;
13209
13210 static u_char* pusha_code = NULL;
13211 static int pusha_len = 0;
13212
13213 static u_char* vzup_code = NULL;
13214 static int vzup_len = 0;
13215
13216 void Assembler::precompute_instructions() {
13217 assert(!Universe::is_fully_initialized(), "must still be single threaded");
13218 guarantee(!precomputed, "only once");
13219 precomputed = true;
13220 ResourceMark rm;
13221
13222 // Make a temporary buffer big enough for the routines we're capturing
13223 int size = 256;
13224 char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
13225 CodeBuffer buffer((address)tmp_code, size);
13226 MacroAssembler masm(&buffer);
13227
13228 address begin_popa = masm.code_section()->end();
13229 masm.popa_uncached();
13230 address end_popa = masm.code_section()->end();
13231 masm.pusha_uncached();
13232 address end_pusha = masm.code_section()->end();
13233 masm.vzeroupper_uncached();
13234 address end_vzup = masm.code_section()->end();
13235
13236 // Save the instructions to permanent buffers.
13237 popa_len = (int)(end_popa - begin_popa);
13238 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
13239 memcpy(popa_code, begin_popa, popa_len);
13240
13241 pusha_len = (int)(end_pusha - end_popa);
13242 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
13243 memcpy(pusha_code, end_popa, pusha_len);
13244
13245 vzup_len = (int)(end_vzup - end_pusha);
13246 if (vzup_len > 0) {
13247 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
13248 memcpy(vzup_code, end_pusha, vzup_len);
13249 } else {
13250 vzup_code = pusha_code; // dummy
13251 }
13252
13253 assert(masm.code()->total_oop_size() == 0 &&
13254 masm.code()->total_metadata_size() == 0 &&
13255 masm.code()->total_relocation_size() == 0,
13256 "pre-computed code can't reference oops, metadata or contain relocations");
13257 }
13258
13259 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
13260 assert(src != NULL, "code to copy must have been pre-computed");
13261 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
13262 address end = code_section->end();
13263 memcpy(end, src, src_len);
13264 code_section->set_end(end + src_len);
13265 }
13266
13267 void Assembler::popa() { // 64bit
13268 emit_copy(code_section(), popa_code, popa_len);
13269 }
13270
13271 void Assembler::popa_uncached() { // 64bit
13272 movq(r15, Address(rsp, 0));
13273 movq(r14, Address(rsp, wordSize));
13274 movq(r13, Address(rsp, 2 * wordSize));
13275 movq(r12, Address(rsp, 3 * wordSize));
13276 movq(r11, Address(rsp, 4 * wordSize));
13277 movq(r10, Address(rsp, 5 * wordSize));
13278 movq(r9, Address(rsp, 6 * wordSize));
13279 movq(r8, Address(rsp, 7 * wordSize));
13280 movq(rdi, Address(rsp, 8 * wordSize));
13281 movq(rsi, Address(rsp, 9 * wordSize));
13282 movq(rbp, Address(rsp, 10 * wordSize));
13283 // Skip rsp as it is restored automatically to the value
13284 // before the corresponding pusha when popa is done.
13285 movq(rbx, Address(rsp, 12 * wordSize));
13286 movq(rdx, Address(rsp, 13 * wordSize));
13287 movq(rcx, Address(rsp, 14 * wordSize));
13288 movq(rax, Address(rsp, 15 * wordSize));
13289
13290 addq(rsp, 16 * wordSize);
13291 }
13292
13293 // Does not actually store the value of rsp on the stack.
13294 // The slot for rsp just contains an arbitrary value.
13295 void Assembler::pusha() { // 64bit
13296 emit_copy(code_section(), pusha_code, pusha_len);
13297 }
13298
13299 // Does not actually store the value of rsp on the stack.
13300 // The slot for rsp just contains an arbitrary value.
13301 void Assembler::pusha_uncached() { // 64bit
13302 subq(rsp, 16 * wordSize);
13303
13304 movq(Address(rsp, 15 * wordSize), rax);
13305 movq(Address(rsp, 14 * wordSize), rcx);
13306 movq(Address(rsp, 13 * wordSize), rdx);
13307 movq(Address(rsp, 12 * wordSize), rbx);
13308 // Skip rsp as the value is normally not used. There are a few places where
13309 // the original value of rsp needs to be known but that can be computed
13310 // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
13311 movq(Address(rsp, 10 * wordSize), rbp);
13312 movq(Address(rsp, 9 * wordSize), rsi);
13313 movq(Address(rsp, 8 * wordSize), rdi);
13314 movq(Address(rsp, 7 * wordSize), r8);
13315 movq(Address(rsp, 6 * wordSize), r9);
13316 movq(Address(rsp, 5 * wordSize), r10);
13317 movq(Address(rsp, 4 * wordSize), r11);
13318 movq(Address(rsp, 3 * wordSize), r12);
13319 movq(Address(rsp, 2 * wordSize), r13);
13320 movq(Address(rsp, wordSize), r14);
13321 movq(Address(rsp, 0), r15);
13322 }
13323
13324 void Assembler::vzeroupper() {
13325 emit_copy(code_section(), vzup_code, vzup_len);
13326 }
13327
13328 void Assembler::pushq(Address src) {
13329 InstructionMark im(this);
13330 emit_int16(get_prefixq(src), (unsigned char)0xFF);
13331 emit_operand(rsi, src);
13332 }
13333
13334 void Assembler::rclq(Register dst, int imm8) {
13335 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13336 int encode = prefixq_and_encode(dst->encoding());
13337 if (imm8 == 1) {
13338 emit_int16((unsigned char)0xD1, (0xD0 | encode));
13339 } else {
13340 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
13341 }
13342 }
13343
13344 void Assembler::rcrq(Register dst, int imm8) {
13345 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13346 int encode = prefixq_and_encode(dst->encoding());
13347 if (imm8 == 1) {
13348 emit_int16((unsigned char)0xD1, (0xD8 | encode));
13349 } else {
13350 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
13351 }
13352 }
13353
13354 void Assembler::rorxl(Register dst, Register src, int imm8) {
13355 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13356 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13357 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13358 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13359 }
13360
13361 void Assembler::rorxl(Register dst, Address src, int imm8) {
13362 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13363 InstructionMark im(this);
13364 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13365 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13366 emit_int8((unsigned char)0xF0);
13367 emit_operand(dst, src);
13368 emit_int8(imm8);
13369 }
13370
13371 void Assembler::rorxq(Register dst, Register src, int imm8) {
13372 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13373 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13374 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13375 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
13376 }
13377
13378 void Assembler::rorxq(Register dst, Address src, int imm8) {
13379 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
13380 InstructionMark im(this);
13381 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
13382 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
13383 emit_int8((unsigned char)0xF0);
13384 emit_operand(dst, src);
13385 emit_int8(imm8);
13386 }
13387
13388 #ifdef _LP64
13389 void Assembler::salq(Address dst, int imm8) {
13390 InstructionMark im(this);
13391 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13392 if (imm8 == 1) {
13393 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13394 emit_operand(as_Register(4), dst);
13395 }
13396 else {
13397 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13398 emit_operand(as_Register(4), dst);
13399 emit_int8(imm8);
13400 }
13401 }
13402
13403 void Assembler::salq(Address dst) {
13404 InstructionMark im(this);
13405 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13406 emit_operand(as_Register(4), dst);
13407 }
13408
13409 void Assembler::salq(Register dst, int imm8) {
13410 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13411 int encode = prefixq_and_encode(dst->encoding());
13412 if (imm8 == 1) {
13413 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13414 } else {
13415 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13416 }
13417 }
13418
13419 void Assembler::salq(Register dst) {
13420 int encode = prefixq_and_encode(dst->encoding());
13421 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13422 }
13423
13424 void Assembler::sarq(Address dst, int imm8) {
13425 InstructionMark im(this);
13426 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13427 if (imm8 == 1) {
13428 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13429 emit_operand(as_Register(7), dst);
13430 }
13431 else {
13432 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13433 emit_operand(as_Register(7), dst);
13434 emit_int8(imm8);
13435 }
13436 }
13437
13438 void Assembler::sarq(Address dst) {
13439 InstructionMark im(this);
13440 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13441 emit_operand(as_Register(7), dst);
13442 }
13443
13444 void Assembler::sarq(Register dst, int imm8) {
13445 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13446 int encode = prefixq_and_encode(dst->encoding());
13447 if (imm8 == 1) {
13448 emit_int16((unsigned char)0xD1, (0xF8 | encode));
13449 } else {
13450 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
13451 }
13452 }
13453
13454 void Assembler::sarq(Register dst) {
13455 int encode = prefixq_and_encode(dst->encoding());
13456 emit_int16((unsigned char)0xD3, (0xF8 | encode));
13457 }
13458 #endif
13459
13460 void Assembler::sbbq(Address dst, int32_t imm32) {
13461 InstructionMark im(this);
13462 prefixq(dst);
13463 emit_arith_operand(0x81, rbx, dst, imm32);
13464 }
13465
13466 void Assembler::sbbq(Register dst, int32_t imm32) {
13467 (void) prefixq_and_encode(dst->encoding());
13468 emit_arith(0x81, 0xD8, dst, imm32);
13469 }
13470
13471 void Assembler::sbbq(Register dst, Address src) {
13472 InstructionMark im(this);
13473 emit_int16(get_prefixq(src, dst), 0x1B);
13474 emit_operand(dst, src);
13475 }
13476
13477 void Assembler::sbbq(Register dst, Register src) {
13478 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13479 emit_arith(0x1B, 0xC0, dst, src);
13480 }
13481
13482 void Assembler::shlq(Register dst, int imm8) {
13483 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13484 int encode = prefixq_and_encode(dst->encoding());
13485 if (imm8 == 1) {
13486 emit_int16((unsigned char)0xD1, (0xE0 | encode));
13487 } else {
13488 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
13489 }
13490 }
13491
13492 void Assembler::shlq(Register dst) {
13493 int encode = prefixq_and_encode(dst->encoding());
13494 emit_int16((unsigned char)0xD3, (0xE0 | encode));
13495 }
13496
13497 void Assembler::shrq(Register dst, int imm8) {
13498 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13499 int encode = prefixq_and_encode(dst->encoding());
13500 if (imm8 == 1) {
13501 emit_int16((unsigned char)0xD1, (0xE8 | encode));
13502 }
13503 else {
13504 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
13505 }
13506 }
13507
13508 void Assembler::shrq(Register dst) {
13509 int encode = prefixq_and_encode(dst->encoding());
13510 emit_int16((unsigned char)0xD3, 0xE8 | encode);
13511 }
13512
13513 void Assembler::shrq(Address dst) {
13514 InstructionMark im(this);
13515 emit_int16(get_prefixq(dst), (unsigned char)0xD3);
13516 emit_operand(as_Register(5), dst);
13517 }
13518
13519 void Assembler::shrq(Address dst, int imm8) {
13520 InstructionMark im(this);
13521 assert(isShiftCount(imm8 >> 1), "illegal shift count");
13522 if (imm8 == 1) {
13523 emit_int16(get_prefixq(dst), (unsigned char)0xD1);
13524 emit_operand(as_Register(5), dst);
13525 }
13526 else {
13527 emit_int16(get_prefixq(dst), (unsigned char)0xC1);
13528 emit_operand(as_Register(5), dst);
13529 emit_int8(imm8);
13530 }
13531 }
13532
13533 void Assembler::subq(Address dst, int32_t imm32) {
13534 InstructionMark im(this);
13535 prefixq(dst);
13536 emit_arith_operand(0x81, rbp, dst, imm32);
13537 }
13538
13539 void Assembler::subq(Address dst, Register src) {
13540 InstructionMark im(this);
13541 emit_int16(get_prefixq(dst, src), 0x29);
13542 emit_operand(src, dst);
13543 }
13544
13545 void Assembler::subq(Register dst, int32_t imm32) {
13546 (void) prefixq_and_encode(dst->encoding());
13547 emit_arith(0x81, 0xE8, dst, imm32);
13548 }
13549
13550 // Force generation of a 4 byte immediate value even if it fits into 8bit
13551 void Assembler::subq_imm32(Register dst, int32_t imm32) {
13552 (void) prefixq_and_encode(dst->encoding());
13553 emit_arith_imm32(0x81, 0xE8, dst, imm32);
13554 }
13555
13556 void Assembler::subq(Register dst, Address src) {
13557 InstructionMark im(this);
13558 emit_int16(get_prefixq(src, dst), 0x2B);
13559 emit_operand(dst, src);
13560 }
13561
13562 void Assembler::subq(Register dst, Register src) {
13563 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13564 emit_arith(0x2B, 0xC0, dst, src);
13565 }
13566
13567 void Assembler::testq(Address dst, int32_t imm32) {
13568 if (imm32 >= 0) {
13569 testl(dst, imm32);
13570 return;
13571 }
13572 InstructionMark im(this);
13573 emit_int16(get_prefixq(dst), (unsigned char)0xF7);
13574 emit_operand(as_Register(0), dst);
13575 emit_int32(imm32);
13576 }
13577
13578 void Assembler::testq(Register dst, int32_t imm32) {
13579 if (imm32 >= 0) {
13580 testl(dst, imm32);
13581 return;
13582 }
13583 // not using emit_arith because test
13584 // doesn't support sign-extension of
13585 // 8bit operands
13586 if (dst == rax) {
13587 prefix(REX_W);
13588 emit_int8((unsigned char)0xA9);
13589 emit_int32(imm32);
13590 } else {
13591 int encode = dst->encoding();
13592 encode = prefixq_and_encode(encode);
13593 emit_int16((unsigned char)0xF7, (0xC0 | encode));
13594 emit_int32(imm32);
13595 }
13596 }
13597
13598 void Assembler::testq(Register dst, Register src) {
13599 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13600 emit_arith(0x85, 0xC0, dst, src);
13601 }
13602
13603 void Assembler::testq(Register dst, Address src) {
13604 InstructionMark im(this);
13605 emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
13606 emit_operand(dst, src);
13607 }
13608
13609 void Assembler::xaddq(Address dst, Register src) {
13610 InstructionMark im(this);
13611 emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
13612 emit_operand(src, dst);
13613 }
13614
13615 void Assembler::xchgq(Register dst, Address src) {
13616 InstructionMark im(this);
13617 emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
13618 emit_operand(dst, src);
13619 }
13620
13621 void Assembler::xchgq(Register dst, Register src) {
13622 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
13623 emit_int16((unsigned char)0x87, (0xc0 | encode));
13624 }
13625
13626 void Assembler::xorq(Register dst, Register src) {
13627 (void) prefixq_and_encode(dst->encoding(), src->encoding());
13628 emit_arith(0x33, 0xC0, dst, src);
13629 }
13630
13631 void Assembler::xorq(Register dst, Address src) {
13632 InstructionMark im(this);
13633 emit_int16(get_prefixq(src, dst), 0x33);
13634 emit_operand(dst, src);
13635 }
13636
13637 void Assembler::xorq(Register dst, int32_t imm32) {
13638 (void) prefixq_and_encode(dst->encoding());
13639 emit_arith(0x81, 0xF0, dst, imm32);
13640 }
13641
13642 void Assembler::xorq(Address dst, int32_t imm32) {
13643 InstructionMark im(this);
13644 prefixq(dst);
13645 emit_arith_operand(0x81, as_Register(6), dst, imm32);
13646 }
13647
13648 void Assembler::xorq(Address dst, Register src) {
13649 InstructionMark im(this);
13650 emit_int16(get_prefixq(dst, src), 0x31);
13651 emit_operand(src, dst);
13652 }
13653
13654 #endif // !LP64
13655
13656 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
13657 if (VM_Version::supports_evex()) {
13658 _tuple_type = tuple_type;
13659 _input_size_in_bits = input_size_in_bits;
13660 }
13661 }