1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef CPU_X86_C2_MACROASSEMBLER_X86_HPP
26 #define CPU_X86_C2_MACROASSEMBLER_X86_HPP
27
28 // C2_MacroAssembler contains high-level macros for C2
29
30 public:
31 // C2 compiled method's prolog code.
32 void verified_entry(Compile* C, int sp_inc = 0);
33
34 void entry_barrier();
35 Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);
36
37 // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
38 // See full description in macroAssembler_x86.cpp.
39 void fast_lock(Register obj, Register box, Register tmp,
40 Register scr, Register cx1, Register cx2, Register thread,
41 Metadata* method_data);
42 void fast_unlock(Register obj, Register box, Register tmp);
43
44 void fast_lock_lightweight(Register obj, Register box, Register rax_reg,
45 Register t, Register thread);
46 void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
47
48 // Generic instructions support for use in .ad files C2 code generation
49 void vabsnegd(int opcode, XMMRegister dst, XMMRegister src);
50 void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
51 void vabsnegf(int opcode, XMMRegister dst, XMMRegister src);
52 void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
53
54 void pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src,
55 XMMRegister tmp = xnoreg);
56 void vpminmax(int opcode, BasicType elem_bt,
57 XMMRegister dst, XMMRegister src1, XMMRegister src2,
58 int vlen_enc);
59
60 void vminmax_fp(int opcode, BasicType elem_bt,
61 XMMRegister dst, XMMRegister a, XMMRegister b,
62 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
63 int vlen_enc);
64 void evminmax_fp(int opcode, BasicType elem_bt,
65 XMMRegister dst, XMMRegister a, XMMRegister b,
66 KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
67 int vlen_enc);
68
69 void signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one);
70
71 void vector_compress_expand(int opcode, XMMRegister dst, XMMRegister src, KRegister mask,
72 bool merge, BasicType bt, int vec_enc);
73
74 void vector_mask_compress(KRegister dst, KRegister src, Register rtmp1, Register rtmp2, int mask_len);
75
76 void vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
77 void vextendbw(bool sign, XMMRegister dst, XMMRegister src);
78 void vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
79 void vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
80
81 void vshiftd(int opcode, XMMRegister dst, XMMRegister shift);
82 void vshiftd_imm(int opcode, XMMRegister dst, int shift);
83 void vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
84 void vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
85 void vshiftw(int opcode, XMMRegister dst, XMMRegister shift);
86 void vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
87 void vshiftq(int opcode, XMMRegister dst, XMMRegister shift);
88 void vshiftq_imm(int opcode, XMMRegister dst, int shift);
89 void vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
90 void vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
91
92 void vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, int shift, int vector_len);
93 void vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
94
95 void varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
96 void varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
97 void varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister vtmp = xnoreg);
98 void varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
99 void evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
100
101 void insert(BasicType typ, XMMRegister dst, Register val, int idx);
102 void vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx);
103 void vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len);
104 void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
105 void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
106
107 void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
108 void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
109
110 // extract
111 void extract(BasicType typ, Register dst, XMMRegister src, int idx);
112 XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
113 void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex);
114 void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, XMMRegister vtmp = xnoreg);
115 void movsxl(BasicType typ, Register dst);
116
117 // vector test
118 void vectortest(BasicType bt, XMMRegister src1, XMMRegister src2, XMMRegister vtmp, int vlen_in_bytes);
119
120 // Covert B2X
121 void vconvert_b2x(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vlen_enc);
122 #ifdef _LP64
123 void vpbroadcast(BasicType elem_bt, XMMRegister dst, Register src, int vlen_enc);
124 #endif
125
126 // blend
127 void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
128 void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch = noreg);
129 void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
130
131 void load_vector(XMMRegister dst, Address src, int vlen_in_bytes);
132 void load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg);
133
134 void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
135 void load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc);
136
137 void load_constant_vector(BasicType bt, XMMRegister dst, InternalAddress src, int vlen);
138 void load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt);
139
140 // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
141
142 // dst = src1 reduce(op, src2) using vtmp as temps
143 void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
144 #ifdef _LP64
145 void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
146 void genmask(KRegister dst, Register len, Register temp);
147 #endif // _LP64
148
149 // dst = reduce(op, src2) using vtmp as temps
150 void reduce_fp(int opcode, int vlen,
151 XMMRegister dst, XMMRegister src,
152 XMMRegister vtmp1, XMMRegister vtmp2 = xnoreg);
153 void unordered_reduce_fp(int opcode, int vlen,
154 XMMRegister dst, XMMRegister src,
155 XMMRegister vtmp1 = xnoreg, XMMRegister vtmp2 = xnoreg);
156 void reduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
157 void mulreduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
158 void reduceS(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
159 void reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
160 XMMRegister dst, XMMRegister src,
161 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
162 void reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid,
163 XMMRegister dst, XMMRegister src,
164 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
165 private:
166 void reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
167 void reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
168 void unorderedReduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
169 void unorderedReduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
170
171 // Int Reduction
172 void reduce2I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
173 void reduce4I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
174 void reduce8I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
175 void reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
176
177 // Byte Reduction
178 void reduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
179 void reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
180 void reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
181 void reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
182 void mulreduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
183 void mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
184 void mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
185 void mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
186
187 // Short Reduction
188 void reduce4S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
189 void reduce8S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
190 void reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
191 void reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
192
193 // Long Reduction
194 #ifdef _LP64
195 void reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
196 void reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
197 void reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
198 #endif // _LP64
199
200 // Float Reduction
201 void reduce2F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
202 void reduce4F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
203 void reduce8F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
204 void reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
205
206 // Unordered Float Reduction
207 void unorderedReduce2F(int opcode, XMMRegister dst, XMMRegister src);
208 void unorderedReduce4F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
209 void unorderedReduce8F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
210 void unorderedReduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
211
212 // Double Reduction
213 void reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
214 void reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
215 void reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
216
217 // Unordered Double Reduction
218 void unorderedReduce2D(int opcode, XMMRegister dst, XMMRegister src);
219 void unorderedReduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
220 void unorderedReduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
221
222 // Base reduction instruction
223 void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
224 void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
225 void unordered_reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
226 void unordered_reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
227
228 public:
229 #ifdef _LP64
230 void vector_mask_operation_helper(int opc, Register dst, Register tmp, int masklen);
231
232 void vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp, int masklen, int masksize, int vec_enc);
233
234 void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
235 Register tmp, int masklen, BasicType bt, int vec_enc);
236 void vector_long_to_maskvec(XMMRegister dst, Register src, Register rtmp1,
237 Register rtmp2, XMMRegister xtmp, int mask_len, int vec_enc);
238 #endif
239
240 void vector_maskall_operation(KRegister dst, Register src, int mask_len);
241
242 #ifndef _LP64
243 void vector_maskall_operation32(KRegister dst, Register src, KRegister ktmp, int mask_len);
244 #endif
245
246 void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
247 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
248
249 void stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
250 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
251
252 // IndexOf strings.
253 // Small strings are loaded through stack if they cross page boundary.
254 void string_indexof(Register str1, Register str2,
255 Register cnt1, Register cnt2,
256 int int_cnt2, Register result,
257 XMMRegister vec, Register tmp,
258 int ae);
259
260 // IndexOf for constant substrings with size >= 8 elements
261 // which don't need to be loaded through stack.
262 void string_indexofC8(Register str1, Register str2,
263 Register cnt1, Register cnt2,
264 int int_cnt2, Register result,
265 XMMRegister vec, Register tmp,
266 int ae);
267
268 // Smallest code: we don't need to load through stack,
269 // check string tail.
270
271 // helper function for string_compare
272 void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
273 Address::ScaleFactor scale, Address::ScaleFactor scale1,
274 Address::ScaleFactor scale2, Register index, int ae);
275 // Compare strings.
276 void string_compare(Register str1, Register str2,
277 Register cnt1, Register cnt2, Register result,
278 XMMRegister vec1, int ae, KRegister mask = knoreg);
279
280 // Search for Non-ASCII character (Negative byte value) in a byte array,
281 // return index of the first such character, otherwise len.
282 void count_positives(Register ary1, Register len,
283 Register result, Register tmp1,
284 XMMRegister vec1, XMMRegister vec2, KRegister mask1 = knoreg, KRegister mask2 = knoreg);
285
286 // Compare char[] or byte[] arrays.
287 void arrays_equals(bool is_array_equ, Register ary1, Register ary2, Register limit,
288 Register result, Register chr, XMMRegister vec1, XMMRegister vec2,
289 bool is_char, KRegister mask = knoreg, bool expand_ary2 = false);
290
291 void arrays_hashcode(Register str1, Register cnt1, Register result,
292 Register tmp1, Register tmp2, Register tmp3, XMMRegister vnext,
293 XMMRegister vcoef0, XMMRegister vcoef1, XMMRegister vcoef2, XMMRegister vcoef3,
294 XMMRegister vresult0, XMMRegister vresult1, XMMRegister vresult2, XMMRegister vresult3,
295 XMMRegister vtmp0, XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3,
296 BasicType eltype);
297
298 // helper functions for arrays_hashcode
299 int arrays_hashcode_elsize(BasicType eltype);
300 void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
301 void arrays_hashcode_elvload(XMMRegister dst, Address src, BasicType eltype);
302 void arrays_hashcode_elvload(XMMRegister dst, AddressLiteral src, BasicType eltype);
303 void arrays_hashcode_elvcast(XMMRegister dst, BasicType eltype);
304
305 #ifdef _LP64
306 void convertF2I(BasicType dst_bt, BasicType src_bt, Register dst, XMMRegister src);
307 #endif
308
309 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
310 XMMRegister dst, XMMRegister src1, XMMRegister src2,
311 bool merge, int vlen_enc, bool is_varshift = false);
312
313 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
314 XMMRegister dst, XMMRegister src1, Address src2,
315 bool merge, int vlen_enc);
316
317 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
318 XMMRegister src1, int imm8, bool merge, int vlen_enc);
319
320 void masked_op(int ideal_opc, int mask_len, KRegister dst,
321 KRegister src1, KRegister src2);
322
323 void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
324 BasicType from_elem_bt, BasicType to_elem_bt);
325
326 void vector_signed_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
327 BasicType from_elem_bt, BasicType to_elem_bt);
328
329 void vector_cast_int_to_subword(BasicType to_elem_bt, XMMRegister dst, XMMRegister zero,
330 XMMRegister xtmp, Register rscratch, int vec_enc);
331
332 void vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
333 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
334 AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
335
336 void vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
337 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
338 Register rscratch, int vec_enc);
339
340 void vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
341 KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
342 Register rscratch, int vec_enc);
343
344 void vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
345 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral sign_flip,
346 Register rscratch, int vec_enc);
347
348 void vector_castD2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
349 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5,
350 AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
351
352
353 void vector_cast_double_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
354 XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5, Register rscratch,
355 AddressLiteral float_sign_flip, int vec_enc);
356
357 void vector_cast_double_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
358 KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
359 int vec_enc);
360
361 void vector_cast_double_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
362 KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
363 int vec_enc);
364
365 void vector_cast_float_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
366 KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
367 int vec_enc);
368
369 void vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
370 KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
371 int vec_enc);
372
373 void vector_cast_float_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
374 XMMRegister xtmp4, Register rscratch, AddressLiteral float_sign_flip,
375 int vec_enc);
376
377 void vector_crosslane_doubleword_pack_avx(XMMRegister dst, XMMRegister src, XMMRegister zero,
378 XMMRegister xtmp, int index, int vec_enc);
379
380 void vector_mask_cast(XMMRegister dst, XMMRegister src, BasicType dst_bt, BasicType src_bt, int vlen);
381
382 #ifdef _LP64
383 void vector_round_double_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
384 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
385
386 void vector_round_float_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
387 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
388
389 void vector_round_float_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
390 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4);
391
392 void vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src, XMMRegister mask,
393 Register rtmp, Register rscratch, XMMRegister permv, XMMRegister xtmp,
394 BasicType bt, int vec_enc);
395 #endif // _LP64
396
397 void udivI(Register rax, Register divisor, Register rdx);
398 void umodI(Register rax, Register divisor, Register rdx);
399 void udivmodI(Register rax, Register divisor, Register rdx, Register tmp);
400
401 #ifdef _LP64
402 void reverseI(Register dst, Register src, XMMRegister xtmp1,
403 XMMRegister xtmp2, Register rtmp);
404 void reverseL(Register dst, Register src, XMMRegister xtmp1,
405 XMMRegister xtmp2, Register rtmp1, Register rtmp2);
406 void udivL(Register rax, Register divisor, Register rdx);
407 void umodL(Register rax, Register divisor, Register rdx);
408 void udivmodL(Register rax, Register divisor, Register rdx, Register tmp);
409 #endif
410
411 void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
412 bool merge, BasicType bt, int vlen_enc);
413
414 void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, Address src3,
415 bool merge, BasicType bt, int vlen_enc);
416
417 void vector_reverse_bit(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
418 XMMRegister xtmp2, Register rtmp, int vec_enc);
419
420 void vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
421 XMMRegister xtmp, Register rscratch = noreg);
422
423 void vector_reverse_byte(BasicType bt, XMMRegister dst, XMMRegister src, int vec_enc);
424
425 void vector_popcount_int(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
426 XMMRegister xtmp2, Register rtmp, int vec_enc);
427
428 void vector_popcount_long(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
429 XMMRegister xtmp2, Register rtmp, int vec_enc);
430
431 void vector_popcount_short(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
432 XMMRegister xtmp2, Register rtmp, int vec_enc);
433
434 void vector_popcount_byte(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
435 XMMRegister xtmp2, Register rtmp, int vec_enc);
436
437 void vector_popcount_integral(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
438 XMMRegister xtmp2, Register rtmp, int vec_enc);
439
440 void vector_popcount_integral_evex(BasicType bt, XMMRegister dst, XMMRegister src,
441 KRegister mask, bool merge, int vec_enc);
442
443 void vbroadcast(BasicType bt, XMMRegister dst, int imm32, Register rtmp, int vec_enc);
444
445 void vector_reverse_byte64(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
446 XMMRegister xtmp2, Register rtmp, int vec_enc);
447
448 void vector_count_leading_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src,
449 XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
450 KRegister ktmp, Register rtmp, bool merge, int vec_enc);
451
452 void vector_count_leading_zeros_byte_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
453 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
454
455 void vector_count_leading_zeros_short_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
456 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
457
458 void vector_count_leading_zeros_int_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
459 XMMRegister xtmp2, XMMRegister xtmp3, int vec_enc);
460
461 void vector_count_leading_zeros_long_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
462 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
463
464 void vector_count_leading_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
465 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
466
467 void vpadd(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
468
469 void vpsub(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
470
471 void vector_count_trailing_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
472 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, KRegister ktmp,
473 Register rtmp, int vec_enc);
474
475 void vector_swap_nbits(int nbits, int bitmask, XMMRegister dst, XMMRegister src,
476 XMMRegister xtmp1, Register rtmp, int vec_enc);
477
478 void vector_count_trailing_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
479 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
480
481 void vector_signum_avx(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
482 XMMRegister xtmp1, int vec_enc);
483
484 void vector_signum_evex(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
485 KRegister ktmp1, int vec_enc);
486
487 void vmovmask(BasicType elem_bt, XMMRegister dst, Address src, XMMRegister mask, int vec_enc);
488
489 void vmovmask(BasicType elem_bt, Address dst, XMMRegister src, XMMRegister mask, int vec_enc);
490
491 void rearrange_bytes(XMMRegister dst, XMMRegister shuffle, XMMRegister src, XMMRegister xtmp1,
492 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, KRegister ktmp, int vlen_enc);
493
494 void vector_rearrange_int_float(BasicType bt, XMMRegister dst, XMMRegister shuffle,
495 XMMRegister src, int vlen_enc);
496
497
498 void vgather_subword(BasicType elem_ty, XMMRegister dst, Register base, Register idx_base, Register offset,
499 Register mask, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp,
500 Register midx, Register length, int vector_len, int vlen_enc);
501
502 #ifdef _LP64
503 void vgather8b_masked_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
504 Register offset, Register mask, Register midx, Register rtmp, int vlen_enc);
505 #endif
506 void vgather8b_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
507 Register offset, Register rtmp, int vlen_enc);
508
509 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP