1 /*
  2  * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef CPU_X86_C2_MACROASSEMBLER_X86_HPP
 26 #define CPU_X86_C2_MACROASSEMBLER_X86_HPP
 27 
 28 // C2_MacroAssembler contains high-level macros for C2
 29 
 30 public:
 31   // C2 compiled method's prolog code.
 32   void verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub);
 33 
 34   Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);
 35 
 36   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
 37   // See full description in macroAssembler_x86.cpp.
 38   void fast_lock(Register obj, Register box, Register tmp,
 39                  Register scr, Register cx1, Register cx2, Register thread,
 40                  Metadata* method_data);
 41   void fast_unlock(Register obj, Register box, Register tmp);
 42 
 43   void fast_lock_lightweight(Register obj, Register box, Register rax_reg,
 44                              Register t, Register thread);
 45   void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
 46 
 47   // Generic instructions support for use in .ad files C2 code generation
 48   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src);
 49   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
 50   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src);
 51   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
 52 
 53   void pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src,
 54                XMMRegister tmp = xnoreg);
 55   void vpminmax(int opcode, BasicType elem_bt,
 56                 XMMRegister dst, XMMRegister src1, XMMRegister src2,
 57                 int vlen_enc);
 58 
 59   void vpuminmax(int opcode, BasicType elem_bt,
 60                 XMMRegister dst, XMMRegister src1, XMMRegister src2,
 61                 int vlen_enc);
 62 
 63   void vpuminmax(int opcode, BasicType elem_bt,
 64                 XMMRegister dst, XMMRegister src1, Address src2,
 65                 int vlen_enc);
 66 
 67   void vminmax_fp(int opcode, BasicType elem_bt,
 68                   XMMRegister dst, XMMRegister a, XMMRegister b,
 69                   XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
 70                   int vlen_enc);
 71 
 72   void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
 73 
 74   void evminmax_fp(int opcode, BasicType elem_bt,
 75                    XMMRegister dst, XMMRegister a, XMMRegister b,
 76                    KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
 77                    int vlen_enc);
 78 
 79   void signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one);
 80 
 81   void vector_compress_expand(int opcode, XMMRegister dst, XMMRegister src, KRegister mask,
 82                               bool merge, BasicType bt, int vec_enc);
 83 
 84   void vector_mask_compress(KRegister dst, KRegister src, Register rtmp1, Register rtmp2, int mask_len);
 85 
 86   void vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 87   void vextendbw(bool sign, XMMRegister dst, XMMRegister src);
 88   void vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 89   void vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 90 
 91   void vshiftd(int opcode, XMMRegister dst, XMMRegister shift);
 92   void vshiftd_imm(int opcode, XMMRegister dst, int shift);
 93   void vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
 94   void vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
 95   void vshiftw(int opcode, XMMRegister dst, XMMRegister shift);
 96   void vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
 97   void vshiftq(int opcode, XMMRegister dst, XMMRegister shift);
 98   void vshiftq_imm(int opcode, XMMRegister dst, int shift);
 99   void vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
100   void vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
101 
102   void vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, int shift, int vector_len);
103   void vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
104 
105   void varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
106   void varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
107   void varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister vtmp = xnoreg);
108   void varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
109   void evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
110 
111   void insert(BasicType typ, XMMRegister dst, Register val, int idx);
112   void vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx);
113   void vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len);
114   void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
115   void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
116 
117   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
118   void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
119   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len);
120 
121   // extract
122   void extract(BasicType typ, Register dst, XMMRegister src, int idx);
123   XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
124   void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex);
125   void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, XMMRegister vtmp = xnoreg);
126   void movsxl(BasicType typ, Register dst);
127 
128   // vector test
129   void vectortest(BasicType bt, XMMRegister src1, XMMRegister src2, XMMRegister vtmp, int vlen_in_bytes);
130 
131  // Covert B2X
132  void vconvert_b2x(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vlen_enc);
133  void vpbroadcast(BasicType elem_bt, XMMRegister dst, Register src, int vlen_enc);
134 
135   // blend
136   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister    src2, int comparison, int vector_len);
137   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch = noreg);
138   void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
139 
140   void load_vector(BasicType bt, XMMRegister dst, Address        src, int vlen_in_bytes);
141   void load_vector(BasicType bt, XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg);
142 
143   void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
144   void load_vector_mask(KRegister   dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc);
145 
146   void load_constant_vector(BasicType bt, XMMRegister dst, InternalAddress src, int vlen);
147   void load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt);
148 
149   // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
150 
151   // dst = src1  reduce(op, src2) using vtmp as temps
152   void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
153   void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
154   void genmask(KRegister dst, Register len, Register temp);
155 
156   // dst = reduce(op, src2) using vtmp as temps
157   void reduce_fp(int opcode, int vlen,
158                  XMMRegister dst, XMMRegister src,
159                  XMMRegister vtmp1, XMMRegister vtmp2 = xnoreg);
160   void unordered_reduce_fp(int opcode, int vlen,
161                            XMMRegister dst, XMMRegister src,
162                            XMMRegister vtmp1 = xnoreg, XMMRegister vtmp2 = xnoreg);
163   void reduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
164   void mulreduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
165   void reduceS(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
166   void reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
167                          XMMRegister dst, XMMRegister src,
168                          XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
169   void reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid,
170                           XMMRegister dst, XMMRegister src,
171                           XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
172  private:
173   void reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
174   void reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
175   void unorderedReduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
176   void unorderedReduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
177 
178   // Int Reduction
179   void reduce2I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
180   void reduce4I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
181   void reduce8I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
182   void reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
183 
184   // Byte Reduction
185   void reduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
186   void reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
187   void reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
188   void reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
189   void mulreduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
190   void mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
191   void mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
192   void mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
193 
194   // Short Reduction
195   void reduce4S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
196   void reduce8S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
197   void reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
198   void reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
199 
200   // Long Reduction
201   void reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
202   void reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
203   void reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
204 
205   // Float Reduction
206   void reduce2F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
207   void reduce4F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
208   void reduce8F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
209   void reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
210 
211   // Unordered Float Reduction
212   void unorderedReduce2F(int opcode, XMMRegister dst, XMMRegister src);
213   void unorderedReduce4F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
214   void unorderedReduce8F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
215   void unorderedReduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
216 
217   // Double Reduction
218   void reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
219   void reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
220   void reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
221 
222   // Unordered Double Reduction
223   void unorderedReduce2D(int opcode, XMMRegister dst, XMMRegister src);
224   void unorderedReduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
225   void unorderedReduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
226 
227   // Base reduction instruction
228   void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
229   void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
230   void unordered_reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
231   void unordered_reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
232 
233  public:
234   void vector_mask_operation_helper(int opc, Register dst, Register tmp, int masklen);
235 
236   void vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp, int masklen, int masksize, int vec_enc);
237 
238   void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
239                              Register tmp, int masklen, BasicType bt, int vec_enc);
240   void vector_long_to_maskvec(XMMRegister dst, Register src, Register rtmp1,
241                               Register rtmp2, XMMRegister xtmp, int mask_len, int vec_enc);
242 
243   void vector_maskall_operation(KRegister dst, Register src, int mask_len);
244 
245   void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
246                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
247 
248   void stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
249                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
250 
251   // IndexOf strings.
252   // Small strings are loaded through stack if they cross page boundary.
253   void string_indexof(Register str1, Register str2,
254                       Register cnt1, Register cnt2,
255                       int int_cnt2,  Register result,
256                       XMMRegister vec, Register tmp,
257                       int ae);
258 
259   // IndexOf for constant substrings with size >= 8 elements
260   // which don't need to be loaded through stack.
261   void string_indexofC8(Register str1, Register str2,
262                       Register cnt1, Register cnt2,
263                       int int_cnt2,  Register result,
264                       XMMRegister vec, Register tmp,
265                       int ae);
266 
267     // Smallest code: we don't need to load through stack,
268     // check string tail.
269 
270   // helper function for string_compare
271   void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
272                           Address::ScaleFactor scale, Address::ScaleFactor scale1,
273                           Address::ScaleFactor scale2, Register index, int ae);
274   // Compare strings.
275   void string_compare(Register str1, Register str2,
276                       Register cnt1, Register cnt2, Register result,
277                       XMMRegister vec1, int ae, KRegister mask = knoreg);
278 
279   // Search for Non-ASCII character (Negative byte value) in a byte array,
280   // return index of the first such character, otherwise len.
281   void count_positives(Register ary1, Register len,
282                        Register result, Register tmp1,
283                        XMMRegister vec1, XMMRegister vec2, KRegister mask1 = knoreg, KRegister mask2 = knoreg);
284 
285   // Compare char[] or byte[] arrays.
286   void arrays_equals(bool is_array_equ, Register ary1, Register ary2, Register limit,
287                      Register result, Register chr, XMMRegister vec1, XMMRegister vec2,
288                      bool is_char, KRegister mask = knoreg, bool expand_ary2 = false);
289 
290   void arrays_hashcode(Register str1, Register cnt1, Register result,
291                        Register tmp1, Register tmp2, Register tmp3, XMMRegister vnext,
292                        XMMRegister vcoef0, XMMRegister vcoef1, XMMRegister vcoef2, XMMRegister vcoef3,
293                        XMMRegister vresult0, XMMRegister vresult1, XMMRegister vresult2, XMMRegister vresult3,
294                        XMMRegister vtmp0, XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3,
295                        BasicType eltype);
296 
297   // helper functions for arrays_hashcode
298   int arrays_hashcode_elsize(BasicType eltype);
299   void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
300   void arrays_hashcode_elvload(XMMRegister dst, Address src, BasicType eltype);
301   void arrays_hashcode_elvload(XMMRegister dst, AddressLiteral src, BasicType eltype);
302   void arrays_hashcode_elvcast(XMMRegister dst, BasicType eltype);
303 
304   void convertF2I(BasicType dst_bt, BasicType src_bt, Register dst, XMMRegister src);
305 
306   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
307                    XMMRegister dst, XMMRegister src1, XMMRegister src2,
308                    bool merge, int vlen_enc, bool is_varshift = false);
309 
310   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
311                    XMMRegister dst, XMMRegister src1, Address src2,
312                    bool merge, int vlen_enc);
313 
314   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
315                    XMMRegister src1, int imm8, bool merge, int vlen_enc);
316 
317   void masked_op(int ideal_opc, int mask_len, KRegister dst,
318                  KRegister src1, KRegister src2);
319 
320   void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
321                             BasicType from_elem_bt, BasicType to_elem_bt);
322 
323   void vector_signed_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
324                           BasicType from_elem_bt, BasicType to_elem_bt);
325 
326   void vector_cast_int_to_subword(BasicType to_elem_bt, XMMRegister dst, XMMRegister zero,
327                                   XMMRegister xtmp, Register rscratch, int vec_enc);
328 
329   void vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
330                           XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
331                           AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
332 
333   void vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
334                            XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
335                            Register rscratch, int vec_enc);
336 
337   void vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
338                            KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
339                            Register rscratch, int vec_enc);
340 
341   void vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
342                            XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral sign_flip,
343                            Register rscratch, int vec_enc);
344 
345   void vector_castD2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
346                           XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5,
347                           AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
348 
349 
350   void vector_cast_double_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
351                                                    XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5, Register rscratch,
352                                                    AddressLiteral float_sign_flip, int vec_enc);
353 
354   void vector_cast_double_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
355                                                     KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
356                                                     int vec_enc);
357 
358   void vector_cast_double_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
359                                                      KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
360                                                      int vec_enc);
361 
362   void vector_cast_float_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
363                                                    KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
364                                                    int vec_enc);
365 
366   void vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
367                                                     KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
368                                                     int vec_enc);
369 
370   void vector_cast_float_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
371                                                   XMMRegister xtmp4, Register rscratch, AddressLiteral float_sign_flip,
372                                                   int vec_enc);
373 
374   void vector_crosslane_doubleword_pack_avx(XMMRegister dst, XMMRegister src, XMMRegister zero,
375                                             XMMRegister xtmp, int index, int vec_enc);
376 
377   void vector_mask_cast(XMMRegister dst, XMMRegister src, BasicType dst_bt, BasicType src_bt, int vlen);
378 
379   void vector_round_double_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
380                                 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
381 
382   void vector_round_float_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
383                                Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
384 
385   void vector_round_float_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
386                               Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4);
387 
388   void vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src, XMMRegister mask,
389                                    Register rtmp, Register rscratch, XMMRegister permv, XMMRegister xtmp,
390                                    BasicType bt, int vec_enc);
391 
392   void udivI(Register rax, Register divisor, Register rdx);
393   void umodI(Register rax, Register divisor, Register rdx);
394   void udivmodI(Register rax, Register divisor, Register rdx, Register tmp);
395 
396   void reverseI(Register dst, Register src, XMMRegister xtmp1,
397                 XMMRegister xtmp2, Register rtmp);
398   void reverseL(Register dst, Register src, XMMRegister xtmp1,
399                 XMMRegister xtmp2, Register rtmp1, Register rtmp2);
400   void udivL(Register rax, Register divisor, Register rdx);
401   void umodL(Register rax, Register divisor, Register rdx);
402   void udivmodL(Register rax, Register divisor, Register rdx, Register tmp);
403 
404   void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
405                   bool merge, BasicType bt, int vlen_enc);
406 
407   void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, Address src3,
408                   bool merge, BasicType bt, int vlen_enc);
409 
410   void vector_reverse_bit(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
411                           XMMRegister xtmp2, Register rtmp, int vec_enc);
412 
413   void vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
414                                XMMRegister xtmp, Register rscratch = noreg);
415 
416   void vector_reverse_byte(BasicType bt, XMMRegister dst, XMMRegister src, int vec_enc);
417 
418   void vector_popcount_int(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
419                            XMMRegister xtmp2, Register rtmp, int vec_enc);
420 
421   void vector_popcount_long(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
422                             XMMRegister xtmp2, Register rtmp, int vec_enc);
423 
424   void vector_popcount_short(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
425                              XMMRegister xtmp2, Register rtmp, int vec_enc);
426 
427   void vector_popcount_byte(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
428                             XMMRegister xtmp2, Register rtmp, int vec_enc);
429 
430   void vector_popcount_integral(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
431                                 XMMRegister xtmp2, Register rtmp, int vec_enc);
432 
433   void vector_popcount_integral_evex(BasicType bt, XMMRegister dst, XMMRegister src,
434                                      KRegister mask, bool merge, int vec_enc);
435 
436   void vbroadcast(BasicType bt, XMMRegister dst, int imm32, Register rtmp, int vec_enc);
437 
438   void vector_reverse_byte64(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
439                              XMMRegister xtmp2, Register rtmp, int vec_enc);
440 
441   void vector_count_leading_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src,
442                                        XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
443                                        KRegister ktmp, Register rtmp, bool merge, int vec_enc);
444 
445   void vector_count_leading_zeros_byte_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
446                                            XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
447 
448   void vector_count_leading_zeros_short_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
449                                             XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
450 
451   void vector_count_leading_zeros_int_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
452                                           XMMRegister xtmp2, XMMRegister xtmp3, int vec_enc);
453 
454   void vector_count_leading_zeros_long_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
455                                            XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
456 
457   void vector_count_leading_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
458                                       XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
459 
460   void vpadd(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
461 
462   void vpsub(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
463 
464   void vector_count_trailing_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
465                                         XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, KRegister ktmp,
466                                         Register rtmp, int vec_enc);
467 
468   void vector_swap_nbits(int nbits, int bitmask, XMMRegister dst, XMMRegister src,
469                          XMMRegister xtmp1, Register rtmp, int vec_enc);
470 
471   void vector_count_trailing_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
472                                        XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
473 
474   void vector_signum_avx(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
475                          XMMRegister xtmp1, int vec_enc);
476 
477   void vector_signum_evex(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
478                           KRegister ktmp1, int vec_enc);
479 
480   void vmovmask(BasicType elem_bt, XMMRegister dst, Address src, XMMRegister mask, int vec_enc);
481 
482   void vmovmask(BasicType elem_bt, Address dst, XMMRegister src, XMMRegister mask, int vec_enc);
483 
484   void rearrange_bytes(XMMRegister dst, XMMRegister shuffle, XMMRegister src, XMMRegister xtmp1,
485                        XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, KRegister ktmp, int vlen_enc);
486 
487   void vector_rearrange_int_float(BasicType bt, XMMRegister dst, XMMRegister shuffle,
488                                   XMMRegister src, int vlen_enc);
489 
490   void efp16sh(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
491 
492   void vgather_subword(BasicType elem_ty, XMMRegister dst,  Register base, Register idx_base, Register offset,
493                        Register mask, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp,
494                        Register midx, Register length, int vector_len, int vlen_enc);
495 
496   void vgather8b_masked_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
497                                Register offset, Register mask, Register midx, Register rtmp, int vlen_enc);
498 
499   void vgather8b_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
500                               Register offset, Register rtmp, int vlen_enc);
501 
502   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, bool is_unsigned, int vlen_enc);
503 
504   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, bool is_unsigned, int vlen_enc);
505 
506   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
507 
508   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
509 
510   void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
511 
512   void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
513 
514   void vector_sub_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, KRegister ktmp, int vlen_enc);
515 
516   void vector_sub_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
517                                              XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
518 
519   void vector_add_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
520                                               XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp, int vlen_enc);
521 
522   void vector_add_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
523                                              XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, int vlen_enc);
524 
525   void vector_addsub_dq_saturating_avx(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
526                                        XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, int vlen_enc);
527 
528   void vector_addsub_dq_saturating_evex(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
529                                         XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, int vlen_enc);
530 
531   void evpmovd2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
532 
533   void evpmovq2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
534 
535   void vpsign_extend_dq(BasicType etype, XMMRegister dst, XMMRegister src, int vlen_enc);
536 
537   void vpgenmin_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
538 
539   void vpgenmax_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
540 
541   void evpcmpu(BasicType etype, KRegister kmask,  XMMRegister src1, XMMRegister src2, Assembler::ComparisonPredicate cond, int vlen_enc);
542 
543   void vpcmpgt(BasicType etype, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
544 
545   void evpmov_vec_to_mask(BasicType etype, KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
546                           int vlen_enc, bool xtmp2_hold_M1 = false);
547 
548   void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
549                               bool is_unsigned, bool merge, int vlen_enc);
550 
551   void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
552                               bool is_unsigned, bool merge, int vlen_enc);
553 
554   void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
555                               bool merge, int vlen_enc);
556 
557   void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
558                               bool merge, int vlen_enc);
559 
560   void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
561                                        XMMRegister src2, bool merge, int vlen_enc);
562 
563   void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
564                                        Address src2, bool merge, int vlen_enc);
565 
566   void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
567 
568   void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
569 
570   void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
571 
572   void vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
573                           KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
574 
575   void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
576                           KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
577 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP