1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef CPU_X86_MACROASSEMBLER_X86_HPP
26 #define CPU_X86_MACROASSEMBLER_X86_HPP
27
28 #include "asm/assembler.hpp"
29 #include "asm/register.hpp"
30 #include "code/vmreg.inline.hpp"
31 #include "compiler/oopMap.hpp"
32 #include "utilities/macros.hpp"
33 #include "runtime/signature.hpp"
34 #include "runtime/vm_version.hpp"
35 #include "utilities/checkedCast.hpp"
36
37 class ciInlineKlass;
38
39 // MacroAssembler extends Assembler by frequently used macros.
40 //
41 // Instructions for which a 'better' code sequence exists depending
42 // on arguments should also go in here.
43
44 class MacroAssembler: public Assembler {
45 friend class LIR_Assembler;
46 friend class Runtime1; // as_Address()
47
48 public:
49 // Support for VM calls
50 //
51 // This is the base routine called by the different versions of call_VM_leaf. The interpreter
52 // may customize this version by overriding it for its purposes (e.g., to save/restore
53 // additional registers when doing a VM call).
54
55 virtual void call_VM_leaf_base(
56 address entry_point, // the entry point
57 int number_of_arguments // the number of arguments to pop after the call
58 );
59
60 protected:
61 // This is the base routine called by the different versions of call_VM. The interpreter
62 // may customize this version by overriding it for its purposes (e.g., to save/restore
63 // additional registers when doing a VM call).
64 //
65 // call_VM_base returns the register which contains the thread upon return.
66 // If no last_java_sp is specified (noreg) than rsp will be used instead.
67 virtual void call_VM_base( // returns the register containing the thread upon return
68 Register oop_result, // where an oop-result ends up if any; use noreg otherwise
69 Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise
70 address entry_point, // the entry point
71 int number_of_arguments, // the number of arguments (w/o thread) to pop after the call
72 bool check_exceptions // whether to check for pending exceptions after return
73 );
74
75 void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
76
77 public:
78 MacroAssembler(CodeBuffer* code) : Assembler(code) {}
79
80 // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
81 // The implementation is only non-empty for the InterpreterMacroAssembler,
82 // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
83 virtual void check_and_handle_popframe();
84 virtual void check_and_handle_earlyret();
85
86 Address as_Address(AddressLiteral adr);
87 Address as_Address(ArrayAddress adr, Register rscratch);
88
89 // Support for null-checks
90 //
91 // Generates code that causes a null OS exception if the content of reg is null.
92 // If the accessed location is M[reg + offset] and the offset is known, provide the
93 // offset. No explicit code generation is needed if the offset is within a certain
94 // range (0 <= offset <= page_size).
95
96 void null_check(Register reg, int offset = -1);
97 static bool needs_explicit_null_check(intptr_t offset);
98 static bool uses_implicit_null_check(void* address);
99
100 // markWord tests, kills markWord reg
101 void test_markword_is_inline_type(Register markword, Label& is_inline_type);
102
103 // inlineKlass queries, kills temp_reg
104 void test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null = true);
105
106 void test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free);
107 void test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free);
108 void test_field_is_flat(Register flags, Register temp_reg, Label& is_flat);
109 void test_field_has_null_marker(Register flags, Register temp_reg, Label& has_null_marker);
110
111 // Check oops for special arrays, i.e. flat arrays and/or null-free arrays
112 void test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label);
113 void test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array);
114 void test_non_flat_array_oop(Register oop, Register temp_reg, Label& is_non_flat_array);
115 void test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array);
116 void test_non_null_free_array_oop(Register oop, Register temp_reg, Label& is_non_null_free_array);
117
118 // Check array klass layout helper for flat or null-free arrays...
119 void test_flat_array_layout(Register lh, Label& is_flat_array);
120 void test_non_flat_array_layout(Register lh, Label& is_non_flat_array);
121
122 // Required platform-specific helpers for Label::patch_instructions.
123 // They _shadow_ the declarations in AbstractAssembler, which are undefined.
124 void pd_patch_instruction(address branch, address target, const char* file, int line) {
125 unsigned char op = branch[0];
126 assert(op == 0xE8 /* call */ ||
127 op == 0xE9 /* jmp */ ||
128 op == 0xEB /* short jmp */ ||
129 (op & 0xF0) == 0x70 /* short jcc */ ||
130 (op == 0x0F && (branch[1] & 0xF0) == 0x80) /* jcc */ ||
131 (op == 0xC7 && branch[1] == 0xF8) /* xbegin */ ||
132 (op == 0x8D) /* lea */,
133 "Invalid opcode at patch point");
134
135 if (op == 0xEB || (op & 0xF0) == 0x70) {
136 // short offset operators (jmp and jcc)
137 char* disp = (char*) &branch[1];
138 int imm8 = checked_cast<int>(target - (address) &disp[1]);
139 guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset at %s:%d",
140 file == nullptr ? "<null>" : file, line);
141 *disp = (char)imm8;
142 } else {
143 int* disp = (int*) &branch[(op == 0x0F || op == 0xC7 || op == 0x8D) ? 2 : 1];
144 int imm32 = checked_cast<int>(target - (address) &disp[1]);
145 *disp = imm32;
146 }
147 }
148
149 // The following 4 methods return the offset of the appropriate move instruction
150
151 // Support for fast byte/short loading with zero extension (depending on particular CPU)
152 int load_unsigned_byte(Register dst, Address src);
153 int load_unsigned_short(Register dst, Address src);
154
155 // Support for fast byte/short loading with sign extension (depending on particular CPU)
156 int load_signed_byte(Register dst, Address src);
157 int load_signed_short(Register dst, Address src);
158
159 // Support for sign-extension (hi:lo = extend_sign(lo))
160 void extend_sign(Register hi, Register lo);
161
162 // Load and store values by size and signed-ness
163 void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
164 void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
165
166 // Support for inc/dec with optimal instruction selection depending on value
167
168 void increment(Register reg, int value = 1) { incrementq(reg, value); }
169 void decrement(Register reg, int value = 1) { decrementq(reg, value); }
170 void increment(Address dst, int value = 1) { incrementq(dst, value); }
171 void decrement(Address dst, int value = 1) { decrementq(dst, value); }
172
173 void decrementl(Address dst, int value = 1);
174 void decrementl(Register reg, int value = 1);
175
176 void decrementq(Register reg, int value = 1);
177 void decrementq(Address dst, int value = 1);
178
179 void incrementl(Address dst, int value = 1);
180 void incrementl(Register reg, int value = 1);
181
182 void incrementq(Register reg, int value = 1);
183 void incrementq(Address dst, int value = 1);
184
185 void incrementl(AddressLiteral dst, Register rscratch = noreg);
186 void incrementl(ArrayAddress dst, Register rscratch);
187
188 void incrementq(AddressLiteral dst, Register rscratch = noreg);
189
190 void movhlf(XMMRegister dst, XMMRegister src, Register rscratch = noreg);
191
192 // Support optimal SSE move instructions.
193 void movflt(XMMRegister dst, XMMRegister src) {
194 if (dst-> encoding() == src->encoding()) return;
195 if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
196 else { movss (dst, src); return; }
197 }
198 void movflt(XMMRegister dst, Address src) { movss(dst, src); }
199 void movflt(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
200 void movflt(Address dst, XMMRegister src) { movss(dst, src); }
201
202 // Move with zero extension
203 void movfltz(XMMRegister dst, XMMRegister src) { movss(dst, src); }
204
205 void movdbl(XMMRegister dst, XMMRegister src) {
206 if (dst-> encoding() == src->encoding()) return;
207 if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
208 else { movsd (dst, src); return; }
209 }
210
211 void movdbl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
212
213 void movdbl(XMMRegister dst, Address src) {
214 if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
215 else { movlpd(dst, src); return; }
216 }
217 void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
218
219 void flt_to_flt16(Register dst, XMMRegister src, XMMRegister tmp) {
220 // Use separate tmp XMM register because caller may
221 // requires src XMM register to be unchanged (as in x86.ad).
222 vcvtps2ph(tmp, src, 0x04, Assembler::AVX_128bit);
223 movdl(dst, tmp);
224 movswl(dst, dst);
225 }
226
227 void flt16_to_flt(XMMRegister dst, Register src) {
228 movdl(dst, src);
229 vcvtph2ps(dst, dst, Assembler::AVX_128bit);
230 }
231
232 // Alignment
233 void align32();
234 void align64();
235 void align(uint modulus);
236 void align(uint modulus, uint target);
237
238 void post_call_nop();
239
240 // Stack frame creation/removal
241 void enter();
242 void leave();
243
244 // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information).
245 // The pointer will be loaded into the thread register. This is a slow version that does native call.
246 // Normally, JavaThread pointer is available in r15_thread, use that where possible.
247 void get_thread_slow(Register thread);
248
249 // Support for argument shuffling
250
251 // bias in bytes
252 void move32_64(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
253 void long_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
254 void float_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
255 void double_move(VMRegPair src, VMRegPair dst, Register tmp = rax, int in_stk_bias = 0, int out_stk_bias = 0);
256 void move_ptr(VMRegPair src, VMRegPair dst);
257 void object_move(OopMap* map,
258 int oop_handle_offset,
259 int framesize_in_slots,
260 VMRegPair src,
261 VMRegPair dst,
262 bool is_receiver,
263 int* receiver_offset);
264
265 // Support for VM calls
266 //
267 // It is imperative that all calls into the VM are handled via the call_VM macros.
268 // They make sure that the stack linkage is setup correctly. call_VM's correspond
269 // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
270
271
272 void call_VM(Register oop_result,
273 address entry_point,
274 bool check_exceptions = true);
275 void call_VM(Register oop_result,
276 address entry_point,
277 Register arg_1,
278 bool check_exceptions = true);
279 void call_VM(Register oop_result,
280 address entry_point,
281 Register arg_1, Register arg_2,
282 bool check_exceptions = true);
283 void call_VM(Register oop_result,
284 address entry_point,
285 Register arg_1, Register arg_2, Register arg_3,
286 bool check_exceptions = true);
287
288 // Overloadings with last_Java_sp
289 void call_VM(Register oop_result,
290 Register last_java_sp,
291 address entry_point,
292 int number_of_arguments = 0,
293 bool check_exceptions = true);
294 void call_VM(Register oop_result,
295 Register last_java_sp,
296 address entry_point,
297 Register arg_1, bool
298 check_exceptions = true);
299 void call_VM(Register oop_result,
300 Register last_java_sp,
301 address entry_point,
302 Register arg_1, Register arg_2,
303 bool check_exceptions = true);
304 void call_VM(Register oop_result,
305 Register last_java_sp,
306 address entry_point,
307 Register arg_1, Register arg_2, Register arg_3,
308 bool check_exceptions = true);
309
310 void get_vm_result_oop(Register oop_result);
311 void get_vm_result_metadata(Register metadata_result);
312
313 // These always tightly bind to MacroAssembler::call_VM_base
314 // bypassing the virtual implementation
315 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
316 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
317 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
318 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
319 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
320
321 void call_VM_leaf0(address entry_point);
322 void call_VM_leaf(address entry_point,
323 int number_of_arguments = 0);
324 void call_VM_leaf(address entry_point,
325 Register arg_1);
326 void call_VM_leaf(address entry_point,
327 Register arg_1, Register arg_2);
328 void call_VM_leaf(address entry_point,
329 Register arg_1, Register arg_2, Register arg_3);
330
331 void call_VM_leaf(address entry_point,
332 Register arg_1, Register arg_2, Register arg_3, Register arg_4);
333
334 // These always tightly bind to MacroAssembler::call_VM_leaf_base
335 // bypassing the virtual implementation
336 void super_call_VM_leaf(address entry_point);
337 void super_call_VM_leaf(address entry_point, Register arg_1);
338 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
339 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
340 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
341
342 void set_last_Java_frame(Register last_java_sp,
343 Register last_java_fp,
344 address last_java_pc,
345 Register rscratch);
346
347 void set_last_Java_frame(Register last_java_sp,
348 Register last_java_fp,
349 Label &last_java_pc,
350 Register scratch);
351
352 void reset_last_Java_frame(bool clear_fp);
353
354 // jobjects
355 void clear_jobject_tag(Register possibly_non_local);
356 void resolve_jobject(Register value, Register tmp);
357 void resolve_global_jobject(Register value, Register tmp);
358
359 // C 'boolean' to Java boolean: x == 0 ? 0 : 1
360 void c2bool(Register x);
361
362 // C++ bool manipulation
363
364 void movbool(Register dst, Address src);
365 void movbool(Address dst, bool boolconst);
366 void movbool(Address dst, Register src);
367 void testbool(Register dst);
368
369 void resolve_oop_handle(Register result, Register tmp);
370 void resolve_weak_handle(Register result, Register tmp);
371 void load_mirror(Register mirror, Register method, Register tmp);
372 void load_method_holder_cld(Register rresult, Register rmethod);
373
374 void load_method_holder(Register holder, Register method);
375
376 // oop manipulations
377
378 // Load oopDesc._metadata without decode (useful for direct Klass* compare from oops)
379 void load_metadata(Register dst, Register src);
380 void load_narrow_klass_compact(Register dst, Register src);
381 void load_klass(Register dst, Register src, Register tmp);
382 void store_klass(Register dst, Register src, Register tmp);
383
384 // Compares the narrow Klass pointer of an object to a given narrow Klass.
385 void cmp_klass(Register klass, Register obj, Register tmp);
386
387 // Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
388 // Uses tmp1 and tmp2 as temporary registers.
389 void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2);
390
391 void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
392 Register tmp1);
393 void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
394 Register tmp1, Register tmp2, Register tmp3);
395
396 void flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info);
397
398 // inline type data payload offsets...
399 void payload_offset(Register inline_klass, Register offset);
400 void payload_addr(Register oop, Register data, Register inline_klass);
401 // get data payload ptr a flat value array at index, kills rcx and index
402 void data_for_value_array_index(Register array, Register array_klass,
403 Register index, Register data);
404
405 void load_heap_oop(Register dst, Address src, Register tmp1 = noreg, DecoratorSet decorators = 0);
406 void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg, DecoratorSet decorators = 0);
407 void store_heap_oop(Address dst, Register val, Register tmp1 = noreg,
408 Register tmp2 = noreg, Register tmp3 = noreg, DecoratorSet decorators = 0);
409
410 // Used for storing null. All other oop constants should be
411 // stored using routines that take a jobject.
412 void store_heap_oop_null(Address dst);
413
414 void load_prototype_header(Register dst, Register src, Register tmp);
415
416 void store_klass_gap(Register dst, Register src);
417
418 // This dummy is to prevent a call to store_heap_oop from
419 // converting a zero (like null) into a Register by giving
420 // the compiler two choices it can't resolve
421
422 void store_heap_oop(Address dst, void* dummy);
423
424 void encode_heap_oop(Register r);
425 void decode_heap_oop(Register r);
426 void encode_heap_oop_not_null(Register r);
427 void decode_heap_oop_not_null(Register r);
428 void encode_heap_oop_not_null(Register dst, Register src);
429 void decode_heap_oop_not_null(Register dst, Register src);
430
431 void set_narrow_oop(Register dst, jobject obj);
432 void set_narrow_oop(Address dst, jobject obj);
433 void cmp_narrow_oop(Register dst, jobject obj);
434 void cmp_narrow_oop(Address dst, jobject obj);
435
436 void encode_klass_not_null(Register r, Register tmp);
437 void decode_klass_not_null(Register r, Register tmp);
438 void encode_and_move_klass_not_null(Register dst, Register src);
439 void decode_and_move_klass_not_null(Register dst, Register src);
440 void set_narrow_klass(Register dst, Klass* k);
441 void set_narrow_klass(Address dst, Klass* k);
442 void cmp_narrow_klass(Register dst, Klass* k);
443 void cmp_narrow_klass(Address dst, Klass* k);
444
445 // if heap base register is used - reinit it with the correct value
446 void reinit_heapbase();
447
448 DEBUG_ONLY(void verify_heapbase(const char* msg);)
449
450 // Int division/remainder for Java
451 // (as idivl, but checks for special case as described in JVM spec.)
452 // returns idivl instruction offset for implicit exception handling
453 int corrected_idivl(Register reg);
454
455 // Long division/remainder for Java
456 // (as idivq, but checks for special case as described in JVM spec.)
457 // returns idivq instruction offset for implicit exception handling
458 int corrected_idivq(Register reg);
459
460 void int3();
461
462 // Long operation macros for a 32bit cpu
463 // Long negation for Java
464 void lneg(Register hi, Register lo);
465
466 // Long multiplication for Java
467 // (destroys contents of eax, ebx, ecx and edx)
468 void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
469
470 // Long shifts for Java
471 // (semantics as described in JVM spec.)
472 void lshl(Register hi, Register lo); // hi:lo << (rcx & 0x3f)
473 void lshr(Register hi, Register lo, bool sign_extension = false); // hi:lo >> (rcx & 0x3f)
474
475 // Long compare for Java
476 // (semantics as described in JVM spec.)
477 void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
478
479
480 // misc
481
482 // Sign extension
483 void sign_extend_short(Register reg);
484 void sign_extend_byte(Register reg);
485
486 // Clean up a subword typed value to the representation in compliance with JVMS ยง2.3
487 void narrow_subword_type(Register reg, BasicType bt);
488
489 // Division by power of 2, rounding towards 0
490 void division_with_shift(Register reg, int shift_value);
491
492 // dst = c = a * b + c
493 void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
494 void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
495
496 void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
497 void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
498 void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
499 void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
500
501
502 // same as fcmp2int, but using SSE2
503 void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
504 void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
505
506 void push_IU_state();
507 void pop_IU_state();
508
509 void push_FPU_state();
510 void pop_FPU_state();
511
512 void push_CPU_state();
513 void pop_CPU_state();
514
515 void push_cont_fastpath();
516 void pop_cont_fastpath();
517
518 DEBUG_ONLY(void stop_if_in_cont(Register cont_reg, const char* name);)
519
520 // Round up to a power of two
521 void round_to(Register reg, int modulus);
522
523 private:
524 // General purpose and XMM registers potentially clobbered by native code; there
525 // is no need for FPU or AVX opmask related methods because C1/interpreter
526 // - we save/restore FPU state as a whole always
527 // - do not care about AVX-512 opmask
528 static RegSet call_clobbered_gp_registers();
529 static XMMRegSet call_clobbered_xmm_registers();
530
531 void push_set(XMMRegSet set, int offset);
532 void pop_set(XMMRegSet set, int offset);
533
534 public:
535 void push_set(RegSet set, int offset = -1);
536 void pop_set(RegSet set, int offset = -1);
537
538 // Push and pop everything that might be clobbered by a native
539 // runtime call.
540 // Only save the lower 64 bits of each vector register.
541 // Additional registers can be excluded in a passed RegSet.
542 void push_call_clobbered_registers_except(RegSet exclude, bool save_fpu = true);
543 void pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu = true);
544
545 void push_call_clobbered_registers(bool save_fpu = true) {
546 push_call_clobbered_registers_except(RegSet(), save_fpu);
547 }
548 void pop_call_clobbered_registers(bool restore_fpu = true) {
549 pop_call_clobbered_registers_except(RegSet(), restore_fpu);
550 }
551
552 // allocation
553 void tlab_allocate(
554 Register obj, // result: pointer to object after successful allocation
555 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
556 int con_size_in_bytes, // object size in bytes if known at compile time
557 Register t1, // temp register
558 Register t2, // temp register
559 Label& slow_case // continuation point if fast allocation fails
560 );
561 void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp);
562
563 void inline_layout_info(Register klass, Register index, Register layout_info);
564
565 void population_count(Register dst, Register src, Register scratch1, Register scratch2);
566
567 // interface method calling
568 void lookup_interface_method(Register recv_klass,
569 Register intf_klass,
570 RegisterOrConstant itable_index,
571 Register method_result,
572 Register scan_temp,
573 Label& no_such_interface,
574 bool return_method = true);
575
576 void lookup_interface_method_stub(Register recv_klass,
577 Register holder_klass,
578 Register resolved_klass,
579 Register method_result,
580 Register scan_temp,
581 Register temp_reg2,
582 Register receiver,
583 int itable_index,
584 Label& L_no_such_interface);
585
586 // virtual method calling
587 void lookup_virtual_method(Register recv_klass,
588 RegisterOrConstant vtable_index,
589 Register method_result);
590
591 // Test sub_klass against super_klass, with fast and slow paths.
592
593 // The fast path produces a tri-state answer: yes / no / maybe-slow.
594 // One of the three labels can be null, meaning take the fall-through.
595 // If super_check_offset is -1, the value is loaded up from super_klass.
596 // No registers are killed, except temp_reg.
597 void check_klass_subtype_fast_path(Register sub_klass,
598 Register super_klass,
599 Register temp_reg,
600 Label* L_success,
601 Label* L_failure,
602 Label* L_slow_path,
603 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
604
605 // The rest of the type check; must be wired to a corresponding fast path.
606 // It does not repeat the fast path logic, so don't use it standalone.
607 // The temp_reg and temp2_reg can be noreg, if no temps are available.
608 // Updates the sub's secondary super cache as necessary.
609 // If set_cond_codes, condition codes will be Z on success, NZ on failure.
610 void check_klass_subtype_slow_path(Register sub_klass,
611 Register super_klass,
612 Register temp_reg,
613 Register temp2_reg,
614 Label* L_success,
615 Label* L_failure,
616 bool set_cond_codes = false);
617
618 // The 64-bit version, which may do a hashed subclass lookup.
619 void check_klass_subtype_slow_path(Register sub_klass,
620 Register super_klass,
621 Register temp_reg,
622 Register temp2_reg,
623 Register temp3_reg,
624 Register temp4_reg,
625 Label* L_success,
626 Label* L_failure);
627
628 // Three parts of a hashed subclass lookup: a simple linear search,
629 // a table lookup, and a fallback that does linear probing in the
630 // event of a hash collision.
631 void check_klass_subtype_slow_path_linear(Register sub_klass,
632 Register super_klass,
633 Register temp_reg,
634 Register temp2_reg,
635 Label* L_success,
636 Label* L_failure,
637 bool set_cond_codes = false);
638 void check_klass_subtype_slow_path_table(Register sub_klass,
639 Register super_klass,
640 Register temp_reg,
641 Register temp2_reg,
642 Register temp3_reg,
643 Register result_reg,
644 Label* L_success,
645 Label* L_failure);
646 void hashed_check_klass_subtype_slow_path(Register sub_klass,
647 Register super_klass,
648 Register temp_reg,
649 Label* L_success,
650 Label* L_failure);
651
652 // As above, but with a constant super_klass.
653 // The result is in Register result, not the condition codes.
654 void lookup_secondary_supers_table_const(Register sub_klass,
655 Register super_klass,
656 Register temp1,
657 Register temp2,
658 Register temp3,
659 Register temp4,
660 Register result,
661 u1 super_klass_slot);
662
663 using Assembler::salq;
664 void salq(Register dest, Register count);
665 using Assembler::rorq;
666 void rorq(Register dest, Register count);
667 void lookup_secondary_supers_table_var(Register sub_klass,
668 Register super_klass,
669 Register temp1,
670 Register temp2,
671 Register temp3,
672 Register temp4,
673 Register result);
674
675 void lookup_secondary_supers_table_slow_path(Register r_super_klass,
676 Register r_array_base,
677 Register r_array_index,
678 Register r_bitmap,
679 Register temp1,
680 Register temp2,
681 Label* L_success,
682 Label* L_failure = nullptr);
683
684 void verify_secondary_supers_table(Register r_sub_klass,
685 Register r_super_klass,
686 Register expected,
687 Register temp1,
688 Register temp2,
689 Register temp3);
690
691 void repne_scanq(Register addr, Register value, Register count, Register limit,
692 Label* L_success,
693 Label* L_failure = nullptr);
694
695 // If r is valid, return r.
696 // If r is invalid, remove a register r2 from available_regs, add r2
697 // to regs_to_push, then return r2.
698 Register allocate_if_noreg(const Register r,
699 RegSetIterator<Register> &available_regs,
700 RegSet ®s_to_push);
701
702 // Simplified, combined version, good for typical uses.
703 // Falls through on failure.
704 void check_klass_subtype(Register sub_klass,
705 Register super_klass,
706 Register temp_reg,
707 Label& L_success);
708
709 void clinit_barrier(Register klass,
710 Label* L_fast_path = nullptr,
711 Label* L_slow_path = nullptr);
712
713 // method handles (JSR 292)
714 Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
715
716 void profile_receiver_type(Register recv, Register mdp, int mdp_offset);
717
718 // Debugging
719
720 // only if +VerifyOops
721 void _verify_oop(Register reg, const char* s, const char* file, int line);
722 void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
723
724 void _verify_oop_checked(Register reg, const char* s, const char* file, int line) {
725 if (VerifyOops) {
726 _verify_oop(reg, s, file, line);
727 }
728 }
729 void _verify_oop_addr_checked(Address reg, const char* s, const char* file, int line) {
730 if (VerifyOops) {
731 _verify_oop_addr(reg, s, file, line);
732 }
733 }
734
735 // TODO: verify method and klass metadata (compare against vptr?)
736 void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
737 void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
738
739 #define verify_oop(reg) _verify_oop_checked(reg, "broken oop " #reg, __FILE__, __LINE__)
740 #define verify_oop_msg(reg, msg) _verify_oop_checked(reg, "broken oop " #reg ", " #msg, __FILE__, __LINE__)
741 #define verify_oop_addr(addr) _verify_oop_addr_checked(addr, "broken oop addr " #addr, __FILE__, __LINE__)
742 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
743 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
744
745 // Verify or restore cpu control state after JNI call
746 void restore_cpu_control_state_after_jni(Register rscratch);
747
748 // prints msg, dumps registers and stops execution
749 void stop(const char* msg);
750
751 // prints msg and continues
752 void warn(const char* msg);
753
754 // dumps registers and other state
755 void print_state();
756
757 static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
758 static void debug64(char* msg, int64_t pc, int64_t regs[]);
759 static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
760 static void print_state64(int64_t pc, int64_t regs[]);
761
762 void os_breakpoint();
763
764 void untested() { stop("untested"); }
765
766 void unimplemented(const char* what = "");
767
768 void should_not_reach_here() { stop("should not reach here"); }
769
770 void print_CPU_state();
771
772 // Stack overflow checking
773 void bang_stack_with_offset(int offset) {
774 // stack grows down, caller passes positive offset
775 assert(offset > 0, "must bang with negative offset");
776 movl(Address(rsp, (-offset)), rax);
777 }
778
779 // Writes to stack successive pages until offset reached to check for
780 // stack overflow + shadow pages. Also, clobbers tmp
781 void bang_stack_size(Register size, Register tmp);
782
783 // Check for reserved stack access in method being exited (for JIT)
784 void reserved_stack_check();
785
786 void safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod);
787
788 void verify_tlab();
789
790 static Condition negate_condition(Condition cond);
791
792 // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
793 // operands. In general the names are modified to avoid hiding the instruction in Assembler
794 // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
795 // here in MacroAssembler. The major exception to this rule is call
796
797 // Arithmetics
798
799
800 void addptr(Address dst, int32_t src) { addq(dst, src); }
801 void addptr(Address dst, Register src);
802
803 void addptr(Register dst, Address src) { addq(dst, src); }
804 void addptr(Register dst, int32_t src);
805 void addptr(Register dst, Register src);
806 void addptr(Register dst, RegisterOrConstant src) {
807 if (src.is_constant()) addptr(dst, checked_cast<int>(src.as_constant()));
808 else addptr(dst, src.as_register());
809 }
810
811 void andptr(Register dst, int32_t src);
812 void andptr(Register src1, Register src2) { andq(src1, src2); }
813 void andptr(Register dst, Address src) { andq(dst, src); }
814
815 using Assembler::andq;
816 void andq(Register dst, AddressLiteral src, Register rscratch = noreg);
817
818 void cmp8(AddressLiteral src1, int imm, Register rscratch = noreg);
819
820 // renamed to drag out the casting of address to int32_t/intptr_t
821 void cmp32(Register src1, int32_t imm);
822
823 void cmp32(AddressLiteral src1, int32_t imm, Register rscratch = noreg);
824 // compare reg - mem, or reg - &mem
825 void cmp32(Register src1, AddressLiteral src2, Register rscratch = noreg);
826
827 void cmp32(Register src1, Address src2);
828
829 void cmpoop(Register src1, Register src2);
830 void cmpoop(Register src1, Address src2);
831 void cmpoop(Register dst, jobject obj, Register rscratch);
832
833 // NOTE src2 must be the lval. This is NOT an mem-mem compare
834 void cmpptr(Address src1, AddressLiteral src2, Register rscratch);
835
836 void cmpptr(Register src1, AddressLiteral src2, Register rscratch = noreg);
837
838 void cmpptr(Register src1, Register src2) { cmpq(src1, src2); }
839 void cmpptr(Register src1, Address src2) { cmpq(src1, src2); }
840
841 void cmpptr(Register src1, int32_t src2) { cmpq(src1, src2); }
842 void cmpptr(Address src1, int32_t src2) { cmpq(src1, src2); }
843
844 // cmp64 to avoild hiding cmpq
845 void cmp64(Register src1, AddressLiteral src, Register rscratch = noreg);
846
847 void cmpxchgptr(Register reg, Address adr);
848
849 void locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch = noreg);
850
851 void imulptr(Register dst, Register src) { imulq(dst, src); }
852 void imulptr(Register dst, Register src, int imm32) { imulq(dst, src, imm32); }
853
854
855 void negptr(Register dst) { negq(dst); }
856
857 void notptr(Register dst) { notq(dst); }
858
859 void shlptr(Register dst, int32_t shift);
860 void shlptr(Register dst) { shlq(dst); }
861
862 void shrptr(Register dst, int32_t shift);
863 void shrptr(Register dst) { shrq(dst); }
864
865 void sarptr(Register dst) { sarq(dst); }
866 void sarptr(Register dst, int32_t src) { sarq(dst, src); }
867
868 void subptr(Address dst, int32_t src) { subq(dst, src); }
869
870 void subptr(Register dst, Address src) { subq(dst, src); }
871 void subptr(Register dst, int32_t src);
872 // Force generation of a 4 byte immediate value even if it fits into 8bit
873 void subptr_imm32(Register dst, int32_t src);
874 void subptr(Register dst, Register src);
875 void subptr(Register dst, RegisterOrConstant src) {
876 if (src.is_constant()) subptr(dst, (int) src.as_constant());
877 else subptr(dst, src.as_register());
878 }
879
880 void sbbptr(Address dst, int32_t src) { sbbq(dst, src); }
881 void sbbptr(Register dst, int32_t src) { sbbq(dst, src); }
882
883 void xchgptr(Register src1, Register src2) { xchgq(src1, src2); }
884 void xchgptr(Register src1, Address src2) { xchgq(src1, src2); }
885
886 void xaddptr(Address src1, Register src2) { xaddq(src1, src2); }
887
888
889
890 // Helper functions for statistics gathering.
891 // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
892 void cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch = noreg);
893 // Unconditional atomic increment.
894 void atomic_incl(Address counter_addr);
895 void atomic_incl(AddressLiteral counter_addr, Register rscratch = noreg);
896 void atomic_incq(Address counter_addr);
897 void atomic_incq(AddressLiteral counter_addr, Register rscratch = noreg);
898 void atomic_incptr(AddressLiteral counter_addr, Register rscratch = noreg) { atomic_incq(counter_addr, rscratch); }
899 void atomic_incptr(Address counter_addr) { atomic_incq(counter_addr); }
900
901 using Assembler::lea;
902 void lea(Register dst, AddressLiteral adr);
903 void lea(Address dst, AddressLiteral adr, Register rscratch);
904
905 void leal32(Register dst, Address src) { leal(dst, src); }
906
907 // Import other testl() methods from the parent class or else
908 // they will be hidden by the following overriding declaration.
909 using Assembler::testl;
910 void testl(Address dst, int32_t imm32);
911 void testl(Register dst, int32_t imm32);
912 void testl(Register dst, AddressLiteral src); // requires reachable address
913 using Assembler::testq;
914 void testq(Address dst, int32_t imm32);
915 void testq(Register dst, int32_t imm32);
916
917 void orptr(Register dst, Address src) { orq(dst, src); }
918 void orptr(Register dst, Register src) { orq(dst, src); }
919 void orptr(Register dst, int32_t src) { orq(dst, src); }
920 void orptr(Address dst, int32_t imm32) { orq(dst, imm32); }
921
922 void testptr(Register src, int32_t imm32) { testq(src, imm32); }
923 void testptr(Register src1, Address src2) { testq(src1, src2); }
924 void testptr(Address src, int32_t imm32) { testq(src, imm32); }
925 void testptr(Register src1, Register src2);
926
927 void xorptr(Register dst, Register src) { xorq(dst, src); }
928 void xorptr(Register dst, Address src) { xorq(dst, src); }
929
930 // Calls
931
932 void call(Label& L, relocInfo::relocType rtype);
933 void call(Register entry);
934 void call(Address addr) { Assembler::call(addr); }
935
936 // NOTE: this call transfers to the effective address of entry NOT
937 // the address contained by entry. This is because this is more natural
938 // for jumps/calls.
939 void call(AddressLiteral entry, Register rscratch = rax);
940
941 // Emit the CompiledIC call idiom
942 void ic_call(address entry, jint method_index = 0);
943 static int ic_check_size();
944 int ic_check(int end_alignment);
945
946 void emit_static_call_stub();
947
948 // Jumps
949
950 // NOTE: these jumps transfer to the effective address of dst NOT
951 // the address contained by dst. This is because this is more natural
952 // for jumps/calls.
953 void jump(AddressLiteral dst, Register rscratch = noreg);
954
955 void jump_cc(Condition cc, AddressLiteral dst, Register rscratch = noreg);
956
957 // 32bit can do a case table jump in one instruction but we no longer allow the base
958 // to be installed in the Address class. This jump will transfer to the address
959 // contained in the location described by entry (not the address of entry)
960 void jump(ArrayAddress entry, Register rscratch);
961
962 // Adding more natural conditional jump instructions
963 void ALWAYSINLINE jo(Label& L, bool maybe_short = true) { jcc(Assembler::overflow, L, maybe_short); }
964 void ALWAYSINLINE jno(Label& L, bool maybe_short = true) { jcc(Assembler::noOverflow, L, maybe_short); }
965 void ALWAYSINLINE js(Label& L, bool maybe_short = true) { jcc(Assembler::negative, L, maybe_short); }
966 void ALWAYSINLINE jns(Label& L, bool maybe_short = true) { jcc(Assembler::positive, L, maybe_short); }
967 void ALWAYSINLINE je(Label& L, bool maybe_short = true) { jcc(Assembler::equal, L, maybe_short); }
968 void ALWAYSINLINE jz(Label& L, bool maybe_short = true) { jcc(Assembler::zero, L, maybe_short); }
969 void ALWAYSINLINE jne(Label& L, bool maybe_short = true) { jcc(Assembler::notEqual, L, maybe_short); }
970 void ALWAYSINLINE jnz(Label& L, bool maybe_short = true) { jcc(Assembler::notZero, L, maybe_short); }
971 void ALWAYSINLINE jb(Label& L, bool maybe_short = true) { jcc(Assembler::below, L, maybe_short); }
972 void ALWAYSINLINE jnae(Label& L, bool maybe_short = true) { jcc(Assembler::below, L, maybe_short); }
973 void ALWAYSINLINE jc(Label& L, bool maybe_short = true) { jcc(Assembler::carrySet, L, maybe_short); }
974 void ALWAYSINLINE jnb(Label& L, bool maybe_short = true) { jcc(Assembler::aboveEqual, L, maybe_short); }
975 void ALWAYSINLINE jae(Label& L, bool maybe_short = true) { jcc(Assembler::aboveEqual, L, maybe_short); }
976 void ALWAYSINLINE jnc(Label& L, bool maybe_short = true) { jcc(Assembler::carryClear, L, maybe_short); }
977 void ALWAYSINLINE jbe(Label& L, bool maybe_short = true) { jcc(Assembler::belowEqual, L, maybe_short); }
978 void ALWAYSINLINE jna(Label& L, bool maybe_short = true) { jcc(Assembler::belowEqual, L, maybe_short); }
979 void ALWAYSINLINE ja(Label& L, bool maybe_short = true) { jcc(Assembler::above, L, maybe_short); }
980 void ALWAYSINLINE jnbe(Label& L, bool maybe_short = true) { jcc(Assembler::above, L, maybe_short); }
981 void ALWAYSINLINE jl(Label& L, bool maybe_short = true) { jcc(Assembler::less, L, maybe_short); }
982 void ALWAYSINLINE jnge(Label& L, bool maybe_short = true) { jcc(Assembler::less, L, maybe_short); }
983 void ALWAYSINLINE jge(Label& L, bool maybe_short = true) { jcc(Assembler::greaterEqual, L, maybe_short); }
984 void ALWAYSINLINE jnl(Label& L, bool maybe_short = true) { jcc(Assembler::greaterEqual, L, maybe_short); }
985 void ALWAYSINLINE jle(Label& L, bool maybe_short = true) { jcc(Assembler::lessEqual, L, maybe_short); }
986 void ALWAYSINLINE jng(Label& L, bool maybe_short = true) { jcc(Assembler::lessEqual, L, maybe_short); }
987 void ALWAYSINLINE jg(Label& L, bool maybe_short = true) { jcc(Assembler::greater, L, maybe_short); }
988 void ALWAYSINLINE jnle(Label& L, bool maybe_short = true) { jcc(Assembler::greater, L, maybe_short); }
989 void ALWAYSINLINE jp(Label& L, bool maybe_short = true) { jcc(Assembler::parity, L, maybe_short); }
990 void ALWAYSINLINE jpe(Label& L, bool maybe_short = true) { jcc(Assembler::parity, L, maybe_short); }
991 void ALWAYSINLINE jnp(Label& L, bool maybe_short = true) { jcc(Assembler::noParity, L, maybe_short); }
992 void ALWAYSINLINE jpo(Label& L, bool maybe_short = true) { jcc(Assembler::noParity, L, maybe_short); }
993 // * No condition for this * void ALWAYSINLINE jcxz(Label& L, bool maybe_short = true) { jcc(Assembler::cxz, L, maybe_short); }
994 // * No condition for this * void ALWAYSINLINE jecxz(Label& L, bool maybe_short = true) { jcc(Assembler::cxz, L, maybe_short); }
995
996 // Short versions of the above
997 void ALWAYSINLINE jo_b(Label& L) { jccb(Assembler::overflow, L); }
998 void ALWAYSINLINE jno_b(Label& L) { jccb(Assembler::noOverflow, L); }
999 void ALWAYSINLINE js_b(Label& L) { jccb(Assembler::negative, L); }
1000 void ALWAYSINLINE jns_b(Label& L) { jccb(Assembler::positive, L); }
1001 void ALWAYSINLINE je_b(Label& L) { jccb(Assembler::equal, L); }
1002 void ALWAYSINLINE jz_b(Label& L) { jccb(Assembler::zero, L); }
1003 void ALWAYSINLINE jne_b(Label& L) { jccb(Assembler::notEqual, L); }
1004 void ALWAYSINLINE jnz_b(Label& L) { jccb(Assembler::notZero, L); }
1005 void ALWAYSINLINE jb_b(Label& L) { jccb(Assembler::below, L); }
1006 void ALWAYSINLINE jnae_b(Label& L) { jccb(Assembler::below, L); }
1007 void ALWAYSINLINE jc_b(Label& L) { jccb(Assembler::carrySet, L); }
1008 void ALWAYSINLINE jnb_b(Label& L) { jccb(Assembler::aboveEqual, L); }
1009 void ALWAYSINLINE jae_b(Label& L) { jccb(Assembler::aboveEqual, L); }
1010 void ALWAYSINLINE jnc_b(Label& L) { jccb(Assembler::carryClear, L); }
1011 void ALWAYSINLINE jbe_b(Label& L) { jccb(Assembler::belowEqual, L); }
1012 void ALWAYSINLINE jna_b(Label& L) { jccb(Assembler::belowEqual, L); }
1013 void ALWAYSINLINE ja_b(Label& L) { jccb(Assembler::above, L); }
1014 void ALWAYSINLINE jnbe_b(Label& L) { jccb(Assembler::above, L); }
1015 void ALWAYSINLINE jl_b(Label& L) { jccb(Assembler::less, L); }
1016 void ALWAYSINLINE jnge_b(Label& L) { jccb(Assembler::less, L); }
1017 void ALWAYSINLINE jge_b(Label& L) { jccb(Assembler::greaterEqual, L); }
1018 void ALWAYSINLINE jnl_b(Label& L) { jccb(Assembler::greaterEqual, L); }
1019 void ALWAYSINLINE jle_b(Label& L) { jccb(Assembler::lessEqual, L); }
1020 void ALWAYSINLINE jng_b(Label& L) { jccb(Assembler::lessEqual, L); }
1021 void ALWAYSINLINE jg_b(Label& L) { jccb(Assembler::greater, L); }
1022 void ALWAYSINLINE jnle_b(Label& L) { jccb(Assembler::greater, L); }
1023 void ALWAYSINLINE jp_b(Label& L) { jccb(Assembler::parity, L); }
1024 void ALWAYSINLINE jpe_b(Label& L) { jccb(Assembler::parity, L); }
1025 void ALWAYSINLINE jnp_b(Label& L) { jccb(Assembler::noParity, L); }
1026 void ALWAYSINLINE jpo_b(Label& L) { jccb(Assembler::noParity, L); }
1027 // * No condition for this * void ALWAYSINLINE jcxz_b(Label& L) { jccb(Assembler::cxz, L); }
1028 // * No condition for this * void ALWAYSINLINE jecxz_b(Label& L) { jccb(Assembler::cxz, L); }
1029
1030 // Floating
1031
1032 void push_f(XMMRegister r);
1033 void pop_f(XMMRegister r);
1034 void push_d(XMMRegister r);
1035 void pop_d(XMMRegister r);
1036
1037 void push_ppx(Register src);
1038 void pop_ppx(Register dst);
1039
1040 void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); }
1041 void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
1042 void andpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1043
1044 void andnpd(XMMRegister dst, XMMRegister src) { Assembler::andnpd(dst, src); }
1045
1046 void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
1047 void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
1048 void andps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1049
1050 void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
1051 void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
1052 void comiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1053
1054 void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
1055 void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
1056 void comisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1057
1058 void orpd(XMMRegister dst, XMMRegister src) { Assembler::orpd(dst, src); }
1059
1060 void cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch = noreg);
1061 void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
1062 void ldmxcsr(AddressLiteral src, Register rscratch = noreg);
1063
1064 private:
1065 void sha256_AVX2_one_round_compute(
1066 Register reg_old_h,
1067 Register reg_a,
1068 Register reg_b,
1069 Register reg_c,
1070 Register reg_d,
1071 Register reg_e,
1072 Register reg_f,
1073 Register reg_g,
1074 Register reg_h,
1075 int iter);
1076 void sha256_AVX2_four_rounds_compute_first(int start);
1077 void sha256_AVX2_four_rounds_compute_last(int start);
1078 void sha256_AVX2_one_round_and_sched(
1079 XMMRegister xmm_0, /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */
1080 XMMRegister xmm_1, /* ymm5 */ /* full cycle is 16 iterations */
1081 XMMRegister xmm_2, /* ymm6 */
1082 XMMRegister xmm_3, /* ymm7 */
1083 Register reg_a, /* == eax on 0 iteration, then rotate 8 register right on each next iteration */
1084 Register reg_b, /* ebx */ /* full cycle is 8 iterations */
1085 Register reg_c, /* edi */
1086 Register reg_d, /* esi */
1087 Register reg_e, /* r8d */
1088 Register reg_f, /* r9d */
1089 Register reg_g, /* r10d */
1090 Register reg_h, /* r11d */
1091 int iter);
1092
1093 void addm(int disp, Register r1, Register r2);
1094
1095 void sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, Register d,
1096 Register e, Register f, Register g, Register h, int iteration);
1097
1098 void sha512_AVX2_one_round_and_schedule(XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1099 Register a, Register b, Register c, Register d, Register e, Register f,
1100 Register g, Register h, int iteration);
1101
1102 void addmq(int disp, Register r1, Register r2);
1103 public:
1104 void sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1105 XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1106 Register buf, Register state, Register ofs, Register limit, Register rsp,
1107 bool multi_block, XMMRegister shuf_mask);
1108 void sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1109 XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1110 Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
1111 XMMRegister shuf_mask);
1112 void sha512_update_ni_x1(Register arg_hash, Register arg_msg, Register ofs, Register limit, bool multi_block);
1113
1114 void fast_md5(Register buf, Address state, Address ofs, Address limit,
1115 bool multi_block);
1116
1117 void fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0,
1118 XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask,
1119 Register buf, Register state, Register ofs, Register limit, Register rsp,
1120 bool multi_block);
1121
1122 void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1123 XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1124 Register buf, Register state, Register ofs, Register limit, Register rsp,
1125 bool multi_block, XMMRegister shuf_mask);
1126
1127 void fast_exp(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1128 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1129 Register rax, Register rcx, Register rdx, Register tmp);
1130
1131 private:
1132
1133 // these are private because users should be doing movflt/movdbl
1134
1135 void movss(Address dst, XMMRegister src) { Assembler::movss(dst, src); }
1136 void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
1137 void movss(XMMRegister dst, Address src) { Assembler::movss(dst, src); }
1138 void movss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1139
1140 void movlpd(XMMRegister dst, Address src) {Assembler::movlpd(dst, src); }
1141 void movlpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1142
1143 public:
1144
1145 void addsd(XMMRegister dst, XMMRegister src) { Assembler::addsd(dst, src); }
1146 void addsd(XMMRegister dst, Address src) { Assembler::addsd(dst, src); }
1147 void addsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1148
1149 void addss(XMMRegister dst, XMMRegister src) { Assembler::addss(dst, src); }
1150 void addss(XMMRegister dst, Address src) { Assembler::addss(dst, src); }
1151 void addss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1152
1153 void addpd(XMMRegister dst, XMMRegister src) { Assembler::addpd(dst, src); }
1154 void addpd(XMMRegister dst, Address src) { Assembler::addpd(dst, src); }
1155 void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1156
1157 using Assembler::vbroadcasti128;
1158 void vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1159
1160 using Assembler::vbroadcastsd;
1161 void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1162
1163 using Assembler::vbroadcastss;
1164 void vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1165
1166 // Vector float blend
1167 void vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg);
1168 void vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg);
1169
1170 void divsd(XMMRegister dst, XMMRegister src) { Assembler::divsd(dst, src); }
1171 void divsd(XMMRegister dst, Address src) { Assembler::divsd(dst, src); }
1172 void divsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1173
1174 void divss(XMMRegister dst, XMMRegister src) { Assembler::divss(dst, src); }
1175 void divss(XMMRegister dst, Address src) { Assembler::divss(dst, src); }
1176 void divss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1177
1178 // Move Unaligned Double Quadword
1179 void movdqu(Address dst, XMMRegister src);
1180 void movdqu(XMMRegister dst, XMMRegister src);
1181 void movdqu(XMMRegister dst, Address src);
1182 void movdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1183
1184 void kmovwl(Register dst, KRegister src) { Assembler::kmovwl(dst, src); }
1185 void kmovwl(Address dst, KRegister src) { Assembler::kmovwl(dst, src); }
1186 void kmovwl(KRegister dst, KRegister src) { Assembler::kmovwl(dst, src); }
1187 void kmovwl(KRegister dst, Register src) { Assembler::kmovwl(dst, src); }
1188 void kmovwl(KRegister dst, Address src) { Assembler::kmovwl(dst, src); }
1189 void kmovwl(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1190
1191 void kmovql(KRegister dst, KRegister src) { Assembler::kmovql(dst, src); }
1192 void kmovql(KRegister dst, Register src) { Assembler::kmovql(dst, src); }
1193 void kmovql(Register dst, KRegister src) { Assembler::kmovql(dst, src); }
1194 void kmovql(KRegister dst, Address src) { Assembler::kmovql(dst, src); }
1195 void kmovql(Address dst, KRegister src) { Assembler::kmovql(dst, src); }
1196 void kmovql(KRegister dst, AddressLiteral src, Register rscratch = noreg);
1197
1198 // Safe move operation, lowers down to 16bit moves for targets supporting
1199 // AVX512F feature and 64bit moves for targets supporting AVX512BW feature.
1200 void kmov(Address dst, KRegister src);
1201 void kmov(KRegister dst, Address src);
1202 void kmov(KRegister dst, KRegister src);
1203 void kmov(Register dst, KRegister src);
1204 void kmov(KRegister dst, Register src);
1205
1206 using Assembler::movddup;
1207 void movddup(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1208
1209 using Assembler::vmovddup;
1210 void vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1211
1212 // AVX Unaligned forms
1213 void vmovdqu(Address dst, XMMRegister src);
1214 void vmovdqu(XMMRegister dst, Address src);
1215 void vmovdqu(XMMRegister dst, XMMRegister src);
1216 void vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1217 void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1218 void vmovdqu(XMMRegister dst, XMMRegister src, int vector_len);
1219 void vmovdqu(XMMRegister dst, Address src, int vector_len);
1220 void vmovdqu(Address dst, XMMRegister src, int vector_len);
1221
1222 // AVX Aligned forms
1223 using Assembler::vmovdqa;
1224 void vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1225 void vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1226
1227 // AVX512 Unaligned
1228 void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
1229 void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
1230 void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len);
1231
1232 void evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1233 void evmovdqub(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
1234
1235 void evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1236 if (dst->encoding() != src->encoding() || mask != k0) {
1237 Assembler::evmovdqub(dst, mask, src, merge, vector_len);
1238 }
1239 }
1240 void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1241 void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1242 void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1243
1244 void evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1245 void evmovdquw(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1246 void evmovdquw(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
1247
1248 void evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1249 if (dst->encoding() != src->encoding() || mask != k0) {
1250 Assembler::evmovdquw(dst, mask, src, merge, vector_len);
1251 }
1252 }
1253 void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1254 void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1255 void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1256
1257 void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
1258 if (dst->encoding() != src->encoding()) {
1259 Assembler::evmovdqul(dst, src, vector_len);
1260 }
1261 }
1262 void evmovdqul(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1263 void evmovdqul(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1264
1265 void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1266 if (dst->encoding() != src->encoding() || mask != k0) {
1267 Assembler::evmovdqul(dst, mask, src, merge, vector_len);
1268 }
1269 }
1270 void evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1271 void evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1272 void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1273
1274 void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
1275 if (dst->encoding() != src->encoding()) {
1276 Assembler::evmovdquq(dst, src, vector_len);
1277 }
1278 }
1279 void evmovdquq(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1280 void evmovdquq(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1281 void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1282 void evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1283
1284 void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1285 if (dst->encoding() != src->encoding() || mask != k0) {
1286 Assembler::evmovdquq(dst, mask, src, merge, vector_len);
1287 }
1288 }
1289 void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1290 void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1291 void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1292 void evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1293
1294 using Assembler::movapd;
1295 void movapd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1296
1297 // Move Aligned Double Quadword
1298 void movdqa(XMMRegister dst, XMMRegister src) { Assembler::movdqa(dst, src); }
1299 void movdqa(XMMRegister dst, Address src) { Assembler::movdqa(dst, src); }
1300 void movdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1301
1302 void movsd(Address dst, XMMRegister src) { Assembler::movsd(dst, src); }
1303 void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
1304 void movsd(XMMRegister dst, Address src) { Assembler::movsd(dst, src); }
1305 void movsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1306
1307 void mulpd(XMMRegister dst, XMMRegister src) { Assembler::mulpd(dst, src); }
1308 void mulpd(XMMRegister dst, Address src) { Assembler::mulpd(dst, src); }
1309 void mulpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1310
1311 void mulsd(XMMRegister dst, XMMRegister src) { Assembler::mulsd(dst, src); }
1312 void mulsd(XMMRegister dst, Address src) { Assembler::mulsd(dst, src); }
1313 void mulsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1314
1315 void mulss(XMMRegister dst, XMMRegister src) { Assembler::mulss(dst, src); }
1316 void mulss(XMMRegister dst, Address src) { Assembler::mulss(dst, src); }
1317 void mulss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1318
1319 // Carry-Less Multiplication Quadword
1320 void pclmulldq(XMMRegister dst, XMMRegister src) {
1321 // 0x00 - multiply lower 64 bits [0:63]
1322 Assembler::pclmulqdq(dst, src, 0x00);
1323 }
1324 void pclmulhdq(XMMRegister dst, XMMRegister src) {
1325 // 0x11 - multiply upper 64 bits [64:127]
1326 Assembler::pclmulqdq(dst, src, 0x11);
1327 }
1328
1329 void pcmpeqb(XMMRegister dst, XMMRegister src);
1330 void pcmpeqw(XMMRegister dst, XMMRegister src);
1331
1332 void pcmpestri(XMMRegister dst, Address src, int imm8);
1333 void pcmpestri(XMMRegister dst, XMMRegister src, int imm8);
1334
1335 void pmovzxbw(XMMRegister dst, XMMRegister src);
1336 void pmovzxbw(XMMRegister dst, Address src);
1337
1338 void pmovmskb(Register dst, XMMRegister src);
1339
1340 void ptest(XMMRegister dst, XMMRegister src);
1341
1342 void roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1343 void roundsd(XMMRegister dst, Address src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
1344 void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch = noreg);
1345
1346 void sqrtss(XMMRegister dst, XMMRegister src) { Assembler::sqrtss(dst, src); }
1347 void sqrtss(XMMRegister dst, Address src) { Assembler::sqrtss(dst, src); }
1348 void sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1349
1350 void subsd(XMMRegister dst, XMMRegister src) { Assembler::subsd(dst, src); }
1351 void subsd(XMMRegister dst, Address src) { Assembler::subsd(dst, src); }
1352 void subsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1353
1354 void subss(XMMRegister dst, XMMRegister src) { Assembler::subss(dst, src); }
1355 void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
1356 void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1357
1358 void evucomish(XMMRegister dst, XMMRegister src) { Assembler::evucomish(dst, src); }
1359 void evucomish(XMMRegister dst, Address src) { Assembler::evucomish(dst, src); }
1360 void evucomish(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1361
1362 void evucomxsh(XMMRegister dst, XMMRegister src) { Assembler::evucomxsh(dst, src); }
1363 void evucomxsh(XMMRegister dst, Address src) { Assembler::evucomxsh(dst, src); }
1364 void evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1365
1366 void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
1367 void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
1368 void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1369
1370 void evucomxss(XMMRegister dst, XMMRegister src) { Assembler::evucomxss(dst, src); }
1371 void evucomxss(XMMRegister dst, Address src) { Assembler::evucomxss(dst, src); }
1372 void evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1373
1374 void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
1375 void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
1376 void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1377
1378 void evucomxsd(XMMRegister dst, XMMRegister src) { Assembler::evucomxsd(dst, src); }
1379 void evucomxsd(XMMRegister dst, Address src) { Assembler::evucomxsd(dst, src); }
1380 void evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1381
1382 // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
1383 void xorpd(XMMRegister dst, XMMRegister src);
1384 void xorpd(XMMRegister dst, Address src) { Assembler::xorpd(dst, src); }
1385 void xorpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1386
1387 // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
1388 void xorps(XMMRegister dst, XMMRegister src);
1389 void xorps(XMMRegister dst, Address src) { Assembler::xorps(dst, src); }
1390 void xorps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1391
1392 // Shuffle Bytes
1393 void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
1394 void pshufb(XMMRegister dst, Address src) { Assembler::pshufb(dst, src); }
1395 void pshufb(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1396 // AVX 3-operands instructions
1397
1398 void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
1399 void vaddsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddsd(dst, nds, src); }
1400 void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1401
1402 void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
1403 void vaddss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddss(dst, nds, src); }
1404 void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1405
1406 void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1407 void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
1408
1409 void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1410 void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1411 void vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1412
1413 void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1414 void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1415
1416 void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1417 void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1418 void vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1419
1420 void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1421 void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1422 void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1423
1424 using Assembler::vpbroadcastd;
1425 void vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1426
1427 using Assembler::vpbroadcastq;
1428 void vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
1429
1430 void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1431 void vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
1432
1433 void vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1434 void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1435 using Assembler::evpcmpeqd;
1436 void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1437
1438 // Vector compares
1439 void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
1440 Assembler::evpcmpd(kdst, mask, nds, src, comparison, is_signed, vector_len);
1441 }
1442 void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1443
1444 void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
1445 Assembler::evpcmpq(kdst, mask, nds, src, comparison, is_signed, vector_len);
1446 }
1447 void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1448
1449 void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
1450 Assembler::evpcmpb(kdst, mask, nds, src, comparison, is_signed, vector_len);
1451 }
1452 void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1453
1454 void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
1455 Assembler::evpcmpw(kdst, mask, nds, src, comparison, is_signed, vector_len);
1456 }
1457 void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
1458
1459 void evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len);
1460
1461 // Emit comparison instruction for the specified comparison predicate.
1462 void vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len);
1463 void vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len);
1464
1465 void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
1466 void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
1467
1468 void vpmovmskb(Register dst, XMMRegister src, int vector_len = Assembler::AVX_256bit);
1469
1470 void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1471 void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1472
1473 void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1474 void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
1475 void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1476
1477 void vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpmuldq(dst, nds, src, vector_len); }
1478
1479 void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1480 void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1481
1482 void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1483 void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1484
1485 void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1486 void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1487
1488 void evpsrad(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1489 void evpsrad(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1490
1491 void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1492 void evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1493
1494 using Assembler::evpsllw;
1495 void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1496 if (!is_varshift) {
1497 Assembler::evpsllw(dst, mask, nds, src, merge, vector_len);
1498 } else {
1499 Assembler::evpsllvw(dst, mask, nds, src, merge, vector_len);
1500 }
1501 }
1502 void evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1503 if (!is_varshift) {
1504 Assembler::evpslld(dst, mask, nds, src, merge, vector_len);
1505 } else {
1506 Assembler::evpsllvd(dst, mask, nds, src, merge, vector_len);
1507 }
1508 }
1509 void evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1510 if (!is_varshift) {
1511 Assembler::evpsllq(dst, mask, nds, src, merge, vector_len);
1512 } else {
1513 Assembler::evpsllvq(dst, mask, nds, src, merge, vector_len);
1514 }
1515 }
1516 void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1517 if (!is_varshift) {
1518 Assembler::evpsrlw(dst, mask, nds, src, merge, vector_len);
1519 } else {
1520 Assembler::evpsrlvw(dst, mask, nds, src, merge, vector_len);
1521 }
1522 }
1523 void evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1524 if (!is_varshift) {
1525 Assembler::evpsrld(dst, mask, nds, src, merge, vector_len);
1526 } else {
1527 Assembler::evpsrlvd(dst, mask, nds, src, merge, vector_len);
1528 }
1529 }
1530
1531 using Assembler::evpsrlq;
1532 void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1533 if (!is_varshift) {
1534 Assembler::evpsrlq(dst, mask, nds, src, merge, vector_len);
1535 } else {
1536 Assembler::evpsrlvq(dst, mask, nds, src, merge, vector_len);
1537 }
1538 }
1539 using Assembler::evpsraw;
1540 void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1541 if (!is_varshift) {
1542 Assembler::evpsraw(dst, mask, nds, src, merge, vector_len);
1543 } else {
1544 Assembler::evpsravw(dst, mask, nds, src, merge, vector_len);
1545 }
1546 }
1547 using Assembler::evpsrad;
1548 void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1549 if (!is_varshift) {
1550 Assembler::evpsrad(dst, mask, nds, src, merge, vector_len);
1551 } else {
1552 Assembler::evpsravd(dst, mask, nds, src, merge, vector_len);
1553 }
1554 }
1555 using Assembler::evpsraq;
1556 void evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
1557 if (!is_varshift) {
1558 Assembler::evpsraq(dst, mask, nds, src, merge, vector_len);
1559 } else {
1560 Assembler::evpsravq(dst, mask, nds, src, merge, vector_len);
1561 }
1562 }
1563
1564 void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1565 void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1566 void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1567 void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1568
1569 void evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1570 void evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1571 void evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1572 void evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1573
1574 void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1575 void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1576
1577 void vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1578 void vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1579
1580 void vptest(XMMRegister dst, XMMRegister src);
1581 void vptest(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vptest(dst, src, vector_len); }
1582
1583 void punpcklbw(XMMRegister dst, XMMRegister src);
1584 void punpcklbw(XMMRegister dst, Address src) { Assembler::punpcklbw(dst, src); }
1585
1586 void pshufd(XMMRegister dst, Address src, int mode);
1587 void pshufd(XMMRegister dst, XMMRegister src, int mode) { Assembler::pshufd(dst, src, mode); }
1588
1589 void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1590 void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
1591
1592 void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1593 void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1594 void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1595
1596 void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1597 void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1598 void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1599
1600 void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1601
1602 void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
1603 void vdivsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivsd(dst, nds, src); }
1604 void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1605
1606 void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
1607 void vdivss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivss(dst, nds, src); }
1608 void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1609
1610 void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
1611 void vmulsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulsd(dst, nds, src); }
1612 void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1613
1614 void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
1615 void vmulss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulss(dst, nds, src); }
1616 void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1617
1618 void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
1619 void vsubsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubsd(dst, nds, src); }
1620 void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1621
1622 void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
1623 void vsubss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubss(dst, nds, src); }
1624 void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1625
1626 void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1627 void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
1628
1629 // AVX Vector instructions
1630
1631 void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1632 void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1633 void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1634
1635 void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1636 void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1637 void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1638
1639 void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1640 if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1641 Assembler::vpxor(dst, nds, src, vector_len);
1642 else
1643 Assembler::vxorpd(dst, nds, src, vector_len);
1644 }
1645 void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
1646 if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1647 Assembler::vpxor(dst, nds, src, vector_len);
1648 else
1649 Assembler::vxorpd(dst, nds, src, vector_len);
1650 }
1651 void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1652
1653 // Simple version for AVX2 256bit vectors
1654 void vpxor(XMMRegister dst, XMMRegister src) {
1655 assert(UseAVX >= 2, "Should be at least AVX2");
1656 Assembler::vpxor(dst, dst, src, AVX_256bit);
1657 }
1658 void vpxor(XMMRegister dst, Address src) {
1659 assert(UseAVX >= 2, "Should be at least AVX2");
1660 Assembler::vpxor(dst, dst, src, AVX_256bit);
1661 }
1662
1663 void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
1664 void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1665
1666 void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
1667 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1668 Assembler::vinserti32x4(dst, nds, src, imm8);
1669 } else if (UseAVX > 1) {
1670 // vinserti128 is available only in AVX2
1671 Assembler::vinserti128(dst, nds, src, imm8);
1672 } else {
1673 Assembler::vinsertf128(dst, nds, src, imm8);
1674 }
1675 }
1676
1677 void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
1678 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1679 Assembler::vinserti32x4(dst, nds, src, imm8);
1680 } else if (UseAVX > 1) {
1681 // vinserti128 is available only in AVX2
1682 Assembler::vinserti128(dst, nds, src, imm8);
1683 } else {
1684 Assembler::vinsertf128(dst, nds, src, imm8);
1685 }
1686 }
1687
1688 void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
1689 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1690 Assembler::vextracti32x4(dst, src, imm8);
1691 } else if (UseAVX > 1) {
1692 // vextracti128 is available only in AVX2
1693 Assembler::vextracti128(dst, src, imm8);
1694 } else {
1695 Assembler::vextractf128(dst, src, imm8);
1696 }
1697 }
1698
1699 void vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
1700 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1701 Assembler::vextracti32x4(dst, src, imm8);
1702 } else if (UseAVX > 1) {
1703 // vextracti128 is available only in AVX2
1704 Assembler::vextracti128(dst, src, imm8);
1705 } else {
1706 Assembler::vextractf128(dst, src, imm8);
1707 }
1708 }
1709
1710 // 128bit copy to/from high 128 bits of 256bit (YMM) vector registers
1711 void vinserti128_high(XMMRegister dst, XMMRegister src) {
1712 vinserti128(dst, dst, src, 1);
1713 }
1714 void vinserti128_high(XMMRegister dst, Address src) {
1715 vinserti128(dst, dst, src, 1);
1716 }
1717 void vextracti128_high(XMMRegister dst, XMMRegister src) {
1718 vextracti128(dst, src, 1);
1719 }
1720 void vextracti128_high(Address dst, XMMRegister src) {
1721 vextracti128(dst, src, 1);
1722 }
1723
1724 void vinsertf128_high(XMMRegister dst, XMMRegister src) {
1725 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1726 Assembler::vinsertf32x4(dst, dst, src, 1);
1727 } else {
1728 Assembler::vinsertf128(dst, dst, src, 1);
1729 }
1730 }
1731
1732 void vinsertf128_high(XMMRegister dst, Address src) {
1733 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1734 Assembler::vinsertf32x4(dst, dst, src, 1);
1735 } else {
1736 Assembler::vinsertf128(dst, dst, src, 1);
1737 }
1738 }
1739
1740 void vextractf128_high(XMMRegister dst, XMMRegister src) {
1741 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1742 Assembler::vextractf32x4(dst, src, 1);
1743 } else {
1744 Assembler::vextractf128(dst, src, 1);
1745 }
1746 }
1747
1748 void vextractf128_high(Address dst, XMMRegister src) {
1749 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1750 Assembler::vextractf32x4(dst, src, 1);
1751 } else {
1752 Assembler::vextractf128(dst, src, 1);
1753 }
1754 }
1755
1756 // 256bit copy to/from high 256 bits of 512bit (ZMM) vector registers
1757 void vinserti64x4_high(XMMRegister dst, XMMRegister src) {
1758 Assembler::vinserti64x4(dst, dst, src, 1);
1759 }
1760 void vinsertf64x4_high(XMMRegister dst, XMMRegister src) {
1761 Assembler::vinsertf64x4(dst, dst, src, 1);
1762 }
1763 void vextracti64x4_high(XMMRegister dst, XMMRegister src) {
1764 Assembler::vextracti64x4(dst, src, 1);
1765 }
1766 void vextractf64x4_high(XMMRegister dst, XMMRegister src) {
1767 Assembler::vextractf64x4(dst, src, 1);
1768 }
1769 void vextractf64x4_high(Address dst, XMMRegister src) {
1770 Assembler::vextractf64x4(dst, src, 1);
1771 }
1772 void vinsertf64x4_high(XMMRegister dst, Address src) {
1773 Assembler::vinsertf64x4(dst, dst, src, 1);
1774 }
1775
1776 // 128bit copy to/from low 128 bits of 256bit (YMM) vector registers
1777 void vinserti128_low(XMMRegister dst, XMMRegister src) {
1778 vinserti128(dst, dst, src, 0);
1779 }
1780 void vinserti128_low(XMMRegister dst, Address src) {
1781 vinserti128(dst, dst, src, 0);
1782 }
1783 void vextracti128_low(XMMRegister dst, XMMRegister src) {
1784 vextracti128(dst, src, 0);
1785 }
1786 void vextracti128_low(Address dst, XMMRegister src) {
1787 vextracti128(dst, src, 0);
1788 }
1789
1790 void vinsertf128_low(XMMRegister dst, XMMRegister src) {
1791 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1792 Assembler::vinsertf32x4(dst, dst, src, 0);
1793 } else {
1794 Assembler::vinsertf128(dst, dst, src, 0);
1795 }
1796 }
1797
1798 void vinsertf128_low(XMMRegister dst, Address src) {
1799 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1800 Assembler::vinsertf32x4(dst, dst, src, 0);
1801 } else {
1802 Assembler::vinsertf128(dst, dst, src, 0);
1803 }
1804 }
1805
1806 void vextractf128_low(XMMRegister dst, XMMRegister src) {
1807 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1808 Assembler::vextractf32x4(dst, src, 0);
1809 } else {
1810 Assembler::vextractf128(dst, src, 0);
1811 }
1812 }
1813
1814 void vextractf128_low(Address dst, XMMRegister src) {
1815 if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1816 Assembler::vextractf32x4(dst, src, 0);
1817 } else {
1818 Assembler::vextractf128(dst, src, 0);
1819 }
1820 }
1821
1822 // 256bit copy to/from low 256 bits of 512bit (ZMM) vector registers
1823 void vinserti64x4_low(XMMRegister dst, XMMRegister src) {
1824 Assembler::vinserti64x4(dst, dst, src, 0);
1825 }
1826 void vinsertf64x4_low(XMMRegister dst, XMMRegister src) {
1827 Assembler::vinsertf64x4(dst, dst, src, 0);
1828 }
1829 void vextracti64x4_low(XMMRegister dst, XMMRegister src) {
1830 Assembler::vextracti64x4(dst, src, 0);
1831 }
1832 void vextractf64x4_low(XMMRegister dst, XMMRegister src) {
1833 Assembler::vextractf64x4(dst, src, 0);
1834 }
1835 void vextractf64x4_low(Address dst, XMMRegister src) {
1836 Assembler::vextractf64x4(dst, src, 0);
1837 }
1838 void vinsertf64x4_low(XMMRegister dst, Address src) {
1839 Assembler::vinsertf64x4(dst, dst, src, 0);
1840 }
1841
1842 // Carry-Less Multiplication Quadword
1843 void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1844 // 0x00 - multiply lower 64 bits [0:63]
1845 Assembler::vpclmulqdq(dst, nds, src, 0x00);
1846 }
1847 void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1848 // 0x11 - multiply upper 64 bits [64:127]
1849 Assembler::vpclmulqdq(dst, nds, src, 0x11);
1850 }
1851 void vpclmullqhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1852 // 0x10 - multiply nds[0:63] and src[64:127]
1853 Assembler::vpclmulqdq(dst, nds, src, 0x10);
1854 }
1855 void vpclmulhqlqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1856 //0x01 - multiply nds[64:127] and src[0:63]
1857 Assembler::vpclmulqdq(dst, nds, src, 0x01);
1858 }
1859
1860 void evpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1861 // 0x00 - multiply lower 64 bits [0:63]
1862 Assembler::evpclmulqdq(dst, nds, src, 0x00, vector_len);
1863 }
1864 void evpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1865 // 0x11 - multiply upper 64 bits [64:127]
1866 Assembler::evpclmulqdq(dst, nds, src, 0x11, vector_len);
1867 }
1868
1869 // AVX-512 mask operations.
1870 void kand(BasicType etype, KRegister dst, KRegister src1, KRegister src2);
1871 void kor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1872 void knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp = knoreg, Register rtmp = noreg);
1873 void kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2);
1874 void kortest(uint masklen, KRegister src1, KRegister src2);
1875 void ktest(uint masklen, KRegister src1, KRegister src2);
1876
1877 void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1878 void evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1879
1880 void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1881 void evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1882
1883 void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1884 void evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1885
1886 void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
1887 void evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
1888
1889 void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1890 void evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1891 void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc);
1892 void evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc);
1893
1894 using Assembler::evpandq;
1895 void evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1896
1897 using Assembler::evpaddq;
1898 void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
1899
1900 using Assembler::evporq;
1901 void evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1902
1903 using Assembler::vpshufb;
1904 void vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1905
1906 using Assembler::vpor;
1907 void vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
1908
1909 using Assembler::vpternlogq;
1910 void vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch = noreg);
1911
1912 void cmov32( Condition cc, Register dst, Address src);
1913 void cmov32( Condition cc, Register dst, Register src);
1914
1915 void cmov( Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
1916
1917 void cmovptr(Condition cc, Register dst, Address src) { cmovq(cc, dst, src); }
1918 void cmovptr(Condition cc, Register dst, Register src) { cmovq(cc, dst, src); }
1919
1920 void movoop(Register dst, jobject obj);
1921 void movoop(Address dst, jobject obj, Register rscratch);
1922
1923 void mov_metadata(Register dst, Metadata* obj);
1924 void mov_metadata(Address dst, Metadata* obj, Register rscratch);
1925
1926 void mov64(Register dst, int64_t imm64);
1927 void mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format);
1928
1929 void movptr(Register dst, Register src);
1930 void movptr(Register dst, Address src);
1931 void movptr(Register dst, AddressLiteral src);
1932 void movptr(Register dst, ArrayAddress src);
1933 void movptr(Register dst, intptr_t src);
1934 void movptr(Address dst, Register src);
1935 void movptr(Address dst, int32_t imm);
1936 void movptr(Address dst, intptr_t src, Register rscratch);
1937 void movptr(ArrayAddress dst, Register src, Register rscratch);
1938
1939 void movptr(Register dst, RegisterOrConstant src) {
1940 if (src.is_constant()) movptr(dst, src.as_constant());
1941 else movptr(dst, src.as_register());
1942 }
1943
1944
1945 // to avoid hiding movl
1946 void mov32(Register dst, AddressLiteral src);
1947 void mov32(AddressLiteral dst, Register src, Register rscratch = noreg);
1948
1949 // Import other mov() methods from the parent class or else
1950 // they will be hidden by the following overriding declaration.
1951 using Assembler::movdl;
1952 void movdl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1953
1954 using Assembler::movq;
1955 void movq(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
1956
1957 // Can push value or effective address
1958 void pushptr(AddressLiteral src, Register rscratch);
1959
1960 void pushptr(Address src) { pushq(src); }
1961 void popptr(Address src) { popq(src); }
1962
1963 void pushoop(jobject obj, Register rscratch);
1964 void pushklass(Metadata* obj, Register rscratch);
1965
1966 // sign extend as need a l to ptr sized element
1967 void movl2ptr(Register dst, Address src) { movslq(dst, src); }
1968 void movl2ptr(Register dst, Register src) { movslq(dst, src); }
1969
1970
1971 public:
1972 // Inline type specific methods
1973 #include "asm/macroAssembler_common.hpp"
1974
1975 // clear memory of size 'cnt' qwords, starting at 'base';
1976 // if 'is_large' is set, do not try to produce short loop
1977 void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only, KRegister mask=knoreg);
1978
1979 // clear memory initialization sequence for constant size;
1980 void clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1981
1982 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
1983 void xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
1984
1985 // Fill primitive arrays
1986 void generate_fill(BasicType t, bool aligned,
1987 Register to, Register value, Register count,
1988 Register rtmp, XMMRegister xtmp);
1989
1990 void encode_iso_array(Register src, Register dst, Register len,
1991 XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1992 XMMRegister tmp4, Register tmp5, Register result, bool ascii);
1993
1994 void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
1995 void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1996 Register y, Register y_idx, Register z,
1997 Register carry, Register product,
1998 Register idx, Register kdx);
1999 void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
2000 Register yz_idx, Register idx,
2001 Register carry, Register product, int offset);
2002 void multiply_128_x_128_bmi2_loop(Register y, Register z,
2003 Register carry, Register carry2,
2004 Register idx, Register jdx,
2005 Register yz_idx1, Register yz_idx2,
2006 Register tmp, Register tmp3, Register tmp4);
2007 void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
2008 Register yz_idx, Register idx, Register jdx,
2009 Register carry, Register product,
2010 Register carry2);
2011 void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
2012 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
2013 void square_rshift(Register x, Register len, Register z, Register tmp1, Register tmp3,
2014 Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
2015 void multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry,
2016 Register tmp2);
2017 void multiply_add_64(Register sum, Register op1, Register op2, Register carry,
2018 Register rdxReg, Register raxReg);
2019 void add_one_64(Register z, Register zlen, Register carry, Register tmp1);
2020 void lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
2021 Register tmp3, Register tmp4);
2022 void square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
2023 Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
2024
2025 void mul_add_128_x_32_loop(Register out, Register in, Register offset, Register len, Register tmp1,
2026 Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
2027 Register raxReg);
2028 void mul_add(Register out, Register in, Register offset, Register len, Register k, Register tmp1,
2029 Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
2030 Register raxReg);
2031 void vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
2032 Register result, Register tmp1, Register tmp2,
2033 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3);
2034
2035 // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
2036 void update_byte_crc32(Register crc, Register val, Register table);
2037 void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
2038
2039 void kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2);
2040 void kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register key, Register pos,
2041 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
2042 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup);
2043
2044 // CRC32C code for java.util.zip.CRC32C::updateBytes() intrinsic
2045 // Note on a naming convention:
2046 // Prefix w = register only used on a Westmere+ architecture
2047 // Prefix n = register only used on a Nehalem architecture
2048 void crc32c_ipl_alg4(Register in_out, uint32_t n,
2049 Register tmp1, Register tmp2, Register tmp3);
2050 void crc32c_pclmulqdq(XMMRegister w_xtmp1,
2051 Register in_out,
2052 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
2053 XMMRegister w_xtmp2,
2054 Register tmp1,
2055 Register n_tmp2, Register n_tmp3);
2056 void crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
2057 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2058 Register tmp1, Register tmp2,
2059 Register n_tmp3);
2060 void crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
2061 Register in_out1, Register in_out2, Register in_out3,
2062 Register tmp1, Register tmp2, Register tmp3,
2063 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2064 Register tmp4, Register tmp5,
2065 Register n_tmp6);
2066 void crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
2067 Register tmp1, Register tmp2, Register tmp3,
2068 Register tmp4, Register tmp5, Register tmp6,
2069 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
2070 bool is_pclmulqdq_supported);
2071 // Fold 128-bit data chunk
2072 void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
2073 void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
2074 // Fold 512-bit data chunk
2075 void fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, Register pos, int offset);
2076 // Fold 8-bit data
2077 void fold_8bit_crc32(Register crc, Register table, Register tmp);
2078 void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
2079
2080 // Compress char[] array to byte[].
2081 void char_array_compress(Register src, Register dst, Register len,
2082 XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
2083 XMMRegister tmp4, Register tmp5, Register result,
2084 KRegister mask1 = knoreg, KRegister mask2 = knoreg);
2085
2086 // Inflate byte[] array to char[].
2087 void byte_array_inflate(Register src, Register dst, Register len,
2088 XMMRegister tmp1, Register tmp2, KRegister mask = knoreg);
2089
2090 void fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
2091 Register length, Register temp, int vec_enc);
2092
2093 void fill64_masked(uint shift, Register dst, int disp,
2094 XMMRegister xmm, KRegister mask, Register length,
2095 Register temp, bool use64byteVector = false);
2096
2097 void fill32_masked(uint shift, Register dst, int disp,
2098 XMMRegister xmm, KRegister mask, Register length,
2099 Register temp);
2100
2101 void fill32(Address dst, XMMRegister xmm);
2102
2103 void fill32(Register dst, int disp, XMMRegister xmm);
2104
2105 void fill64(Address dst, XMMRegister xmm, bool use64byteVector = false);
2106
2107 void fill64(Register dst, int dis, XMMRegister xmm, bool use64byteVector = false);
2108
2109 void convert_f2i(Register dst, XMMRegister src);
2110 void convert_d2i(Register dst, XMMRegister src);
2111 void convert_f2l(Register dst, XMMRegister src);
2112 void convert_d2l(Register dst, XMMRegister src);
2113 void round_double(Register dst, XMMRegister src, Register rtmp, Register rcx);
2114 void round_float(Register dst, XMMRegister src, Register rtmp, Register rcx);
2115
2116 void cache_wb(Address line);
2117 void cache_wbsync(bool is_pre);
2118
2119 #ifdef COMPILER2_OR_JVMCI
2120 void generate_fill_avx3(BasicType type, Register to, Register value,
2121 Register count, Register rtmp, XMMRegister xtmp);
2122 #endif // COMPILER2_OR_JVMCI
2123
2124 void vallones(XMMRegister dst, int vector_len);
2125
2126 void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
2127
2128 void fast_lock(Register basic_lock, Register obj, Register reg_rax, Register tmp, Label& slow);
2129 void fast_unlock(Register obj, Register reg_rax, Register tmp, Label& slow);
2130
2131 void save_legacy_gprs();
2132 void restore_legacy_gprs();
2133 void load_aotrc_address(Register reg, address a);
2134 void setcc(Assembler::Condition comparison, Register dst);
2135 };
2136
2137 #endif // CPU_X86_MACROASSEMBLER_X86_HPP