1 /*
  2  * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef CPU_X86_C2_MACROASSEMBLER_X86_HPP
 26 #define CPU_X86_C2_MACROASSEMBLER_X86_HPP
 27 
 28 // C2_MacroAssembler contains high-level macros for C2
 29 
 30 public:
 31   // C2 compiled method's prolog code.
 32   void verified_entry(Compile* C, int sp_inc = 0);
 33 
 34   void entry_barrier();
 35   Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);
 36 
 37   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
 38   // See full description in macroAssembler_x86.cpp.
 39   void fast_lock(Register obj, Register box, Register tmp,
 40                  Register scr, Register cx1, Register cx2, Register thread,
 41                  Metadata* method_data);
 42   void fast_unlock(Register obj, Register box, Register tmp);
 43 
 44   void fast_lock_lightweight(Register obj, Register box, Register rax_reg,
 45                              Register t, Register thread);
 46   void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
 47 
 48   // Generic instructions support for use in .ad files C2 code generation
 49   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src);
 50   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
 51   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src);
 52   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len);
 53 
 54   void pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src,
 55                XMMRegister tmp = xnoreg);
 56   void vpminmax(int opcode, BasicType elem_bt,
 57                 XMMRegister dst, XMMRegister src1, XMMRegister src2,
 58                 int vlen_enc);
 59 
 60   void vpuminmax(int opcode, BasicType elem_bt,
 61                 XMMRegister dst, XMMRegister src1, XMMRegister src2,
 62                 int vlen_enc);
 63 
 64   void vpuminmax(int opcode, BasicType elem_bt,
 65                 XMMRegister dst, XMMRegister src1, Address src2,
 66                 int vlen_enc);
 67 
 68   void vminmax_fp(int opcode, BasicType elem_bt,
 69                   XMMRegister dst, XMMRegister a, XMMRegister b,
 70                   XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
 71                   int vlen_enc);
 72 
 73   void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
 74 
 75   void evminmax_fp(int opcode, BasicType elem_bt,
 76                    XMMRegister dst, XMMRegister a, XMMRegister b,
 77                    KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
 78                    int vlen_enc);
 79 
 80   void signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one);
 81 
 82   void vector_compress_expand(int opcode, XMMRegister dst, XMMRegister src, KRegister mask,
 83                               bool merge, BasicType bt, int vec_enc);
 84 
 85   void vector_mask_compress(KRegister dst, KRegister src, Register rtmp1, Register rtmp2, int mask_len);
 86 
 87   void vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 88   void vextendbw(bool sign, XMMRegister dst, XMMRegister src);
 89   void vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 90   void vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 91 
 92   void vshiftd(int opcode, XMMRegister dst, XMMRegister shift);
 93   void vshiftd_imm(int opcode, XMMRegister dst, int shift);
 94   void vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
 95   void vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
 96   void vshiftw(int opcode, XMMRegister dst, XMMRegister shift);
 97   void vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
 98   void vshiftq(int opcode, XMMRegister dst, XMMRegister shift);
 99   void vshiftq_imm(int opcode, XMMRegister dst, int shift);
100   void vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
101   void vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
102 
103   void vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, int shift, int vector_len);
104   void vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
105 
106   void varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
107   void varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
108   void varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister vtmp = xnoreg);
109   void varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
110   void evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp);
111 
112   void insert(BasicType typ, XMMRegister dst, Register val, int idx);
113   void vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx);
114   void vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len);
115   void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
116   void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
117 
118   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
119   void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
120   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len);
121 
122   // extract
123   void extract(BasicType typ, Register dst, XMMRegister src, int idx);
124   XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
125   void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex);
126   void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, XMMRegister vtmp = xnoreg);
127   void movsxl(BasicType typ, Register dst);
128 
129   // vector test
130   void vectortest(BasicType bt, XMMRegister src1, XMMRegister src2, XMMRegister vtmp, int vlen_in_bytes);
131 
132  // Covert B2X
133  void vconvert_b2x(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vlen_enc);
134  void vpbroadcast(BasicType elem_bt, XMMRegister dst, Register src, int vlen_enc);
135 
136   // blend
137   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister    src2, int comparison, int vector_len);
138   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch = noreg);
139   void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
140 
141   void load_vector(BasicType bt, XMMRegister dst, Address        src, int vlen_in_bytes);
142   void load_vector(BasicType bt, XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg);
143 
144   void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
145   void load_vector_mask(KRegister   dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc);
146 
147   void load_constant_vector(BasicType bt, XMMRegister dst, InternalAddress src, int vlen);
148   void load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt);
149 
150   // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
151 
152   // dst = src1  reduce(op, src2) using vtmp as temps
153   void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
154   void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
155   void genmask(KRegister dst, Register len, Register temp);
156 
157   // dst = reduce(op, src2) using vtmp as temps
158   void reduce_fp(int opcode, int vlen,
159                  XMMRegister dst, XMMRegister src,
160                  XMMRegister vtmp1, XMMRegister vtmp2 = xnoreg);
161   void unordered_reduce_fp(int opcode, int vlen,
162                            XMMRegister dst, XMMRegister src,
163                            XMMRegister vtmp1 = xnoreg, XMMRegister vtmp2 = xnoreg);
164   void reduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
165   void mulreduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
166   void reduceS(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
167   void reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
168                          XMMRegister dst, XMMRegister src,
169                          XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
170   void reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid,
171                           XMMRegister dst, XMMRegister src,
172                           XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
173  private:
174   void reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
175   void reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
176   void unorderedReduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
177   void unorderedReduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
178 
179   // Int Reduction
180   void reduce2I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
181   void reduce4I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
182   void reduce8I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
183   void reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
184 
185   // Byte Reduction
186   void reduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
187   void reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
188   void reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
189   void reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
190   void mulreduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
191   void mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
192   void mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
193   void mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
194 
195   // Short Reduction
196   void reduce4S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
197   void reduce8S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
198   void reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
199   void reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
200 
201   // Long Reduction
202   void reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
203   void reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
204   void reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
205 
206   // Float Reduction
207   void reduce2F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
208   void reduce4F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
209   void reduce8F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
210   void reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
211 
212   // Unordered Float Reduction
213   void unorderedReduce2F(int opcode, XMMRegister dst, XMMRegister src);
214   void unorderedReduce4F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
215   void unorderedReduce8F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
216   void unorderedReduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
217 
218   // Double Reduction
219   void reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
220   void reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
221   void reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
222 
223   // Unordered Double Reduction
224   void unorderedReduce2D(int opcode, XMMRegister dst, XMMRegister src);
225   void unorderedReduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
226   void unorderedReduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
227 
228   // Base reduction instruction
229   void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
230   void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
231   void unordered_reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
232   void unordered_reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
233 
234  public:
235   void vector_mask_operation_helper(int opc, Register dst, Register tmp, int masklen);
236 
237   void vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp, int masklen, int masksize, int vec_enc);
238 
239   void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
240                              Register tmp, int masklen, BasicType bt, int vec_enc);
241   void vector_long_to_maskvec(XMMRegister dst, Register src, Register rtmp1,
242                               Register rtmp2, XMMRegister xtmp, int mask_len, int vec_enc);
243 
244   void vector_maskall_operation(KRegister dst, Register src, int mask_len);
245 
246   void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
247                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
248 
249   void stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
250                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
251 
252   // IndexOf strings.
253   // Small strings are loaded through stack if they cross page boundary.
254   void string_indexof(Register str1, Register str2,
255                       Register cnt1, Register cnt2,
256                       int int_cnt2,  Register result,
257                       XMMRegister vec, Register tmp,
258                       int ae);
259 
260   // IndexOf for constant substrings with size >= 8 elements
261   // which don't need to be loaded through stack.
262   void string_indexofC8(Register str1, Register str2,
263                       Register cnt1, Register cnt2,
264                       int int_cnt2,  Register result,
265                       XMMRegister vec, Register tmp,
266                       int ae);
267 
268     // Smallest code: we don't need to load through stack,
269     // check string tail.
270 
271   // helper function for string_compare
272   void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
273                           Address::ScaleFactor scale, Address::ScaleFactor scale1,
274                           Address::ScaleFactor scale2, Register index, int ae);
275   // Compare strings.
276   void string_compare(Register str1, Register str2,
277                       Register cnt1, Register cnt2, Register result,
278                       XMMRegister vec1, int ae, KRegister mask = knoreg);
279 
280   // Search for Non-ASCII character (Negative byte value) in a byte array,
281   // return index of the first such character, otherwise len.
282   void count_positives(Register ary1, Register len,
283                        Register result, Register tmp1,
284                        XMMRegister vec1, XMMRegister vec2, KRegister mask1 = knoreg, KRegister mask2 = knoreg);
285 
286   // Compare char[] or byte[] arrays.
287   void arrays_equals(bool is_array_equ, Register ary1, Register ary2, Register limit,
288                      Register result, Register chr, XMMRegister vec1, XMMRegister vec2,
289                      bool is_char, KRegister mask = knoreg, bool expand_ary2 = false);
290 
291   void arrays_hashcode(Register str1, Register cnt1, Register result,
292                        Register tmp1, Register tmp2, Register tmp3, XMMRegister vnext,
293                        XMMRegister vcoef0, XMMRegister vcoef1, XMMRegister vcoef2, XMMRegister vcoef3,
294                        XMMRegister vresult0, XMMRegister vresult1, XMMRegister vresult2, XMMRegister vresult3,
295                        XMMRegister vtmp0, XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3,
296                        BasicType eltype);
297 
298   // helper functions for arrays_hashcode
299   int arrays_hashcode_elsize(BasicType eltype);
300   void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
301   void arrays_hashcode_elvload(XMMRegister dst, Address src, BasicType eltype);
302   void arrays_hashcode_elvload(XMMRegister dst, AddressLiteral src, BasicType eltype);
303   void arrays_hashcode_elvcast(XMMRegister dst, BasicType eltype);
304 
305   void convertF2I(BasicType dst_bt, BasicType src_bt, Register dst, XMMRegister src);
306 
307   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
308                    XMMRegister dst, XMMRegister src1, XMMRegister src2,
309                    bool merge, int vlen_enc, bool is_varshift = false);
310 
311   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
312                    XMMRegister dst, XMMRegister src1, Address src2,
313                    bool merge, int vlen_enc);
314 
315   void evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
316                    XMMRegister src1, int imm8, bool merge, int vlen_enc);
317 
318   void masked_op(int ideal_opc, int mask_len, KRegister dst,
319                  KRegister src1, KRegister src2);
320 
321   void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
322                             BasicType from_elem_bt, BasicType to_elem_bt);
323 
324   void vector_signed_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
325                           BasicType from_elem_bt, BasicType to_elem_bt);
326 
327   void vector_cast_int_to_subword(BasicType to_elem_bt, XMMRegister dst, XMMRegister zero,
328                                   XMMRegister xtmp, Register rscratch, int vec_enc);
329 
330   void vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
331                           XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
332                           AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
333 
334   void vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
335                            XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
336                            Register rscratch, int vec_enc);
337 
338   void vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
339                            KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
340                            Register rscratch, int vec_enc);
341 
342   void vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
343                            XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral sign_flip,
344                            Register rscratch, int vec_enc);
345 
346   void vector_castD2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
347                           XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5,
348                           AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
349 
350 
351   void vector_cast_double_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
352                                                    XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5, Register rscratch,
353                                                    AddressLiteral float_sign_flip, int vec_enc);
354 
355   void vector_cast_double_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
356                                                     KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
357                                                     int vec_enc);
358 
359   void vector_cast_double_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
360                                                      KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
361                                                      int vec_enc);
362 
363   void vector_cast_float_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
364                                                    KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral float_sign_flip,
365                                                    int vec_enc);
366 
367   void vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
368                                                     KRegister ktmp1, KRegister ktmp2, Register rscratch, AddressLiteral double_sign_flip,
369                                                     int vec_enc);
370 
371   void vector_cast_float_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
372                                                   XMMRegister xtmp4, Register rscratch, AddressLiteral float_sign_flip,
373                                                   int vec_enc);
374 
375   void vector_crosslane_doubleword_pack_avx(XMMRegister dst, XMMRegister src, XMMRegister zero,
376                                             XMMRegister xtmp, int index, int vec_enc);
377 
378   void vector_mask_cast(XMMRegister dst, XMMRegister src, BasicType dst_bt, BasicType src_bt, int vlen);
379 
380   void vector_round_double_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
381                                 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
382 
383   void vector_round_float_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
384                                Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
385 
386   void vector_round_float_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
387                               Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4);
388 
389   void vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src, XMMRegister mask,
390                                    Register rtmp, Register rscratch, XMMRegister permv, XMMRegister xtmp,
391                                    BasicType bt, int vec_enc);
392 
393   void udivI(Register rax, Register divisor, Register rdx);
394   void umodI(Register rax, Register divisor, Register rdx);
395   void udivmodI(Register rax, Register divisor, Register rdx, Register tmp);
396 
397   void reverseI(Register dst, Register src, XMMRegister xtmp1,
398                 XMMRegister xtmp2, Register rtmp);
399   void reverseL(Register dst, Register src, XMMRegister xtmp1,
400                 XMMRegister xtmp2, Register rtmp1, Register rtmp2);
401   void udivL(Register rax, Register divisor, Register rdx);
402   void umodL(Register rax, Register divisor, Register rdx);
403   void udivmodL(Register rax, Register divisor, Register rdx, Register tmp);
404 
405   void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
406                   bool merge, BasicType bt, int vlen_enc);
407 
408   void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, Address src3,
409                   bool merge, BasicType bt, int vlen_enc);
410 
411   void vector_reverse_bit(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
412                           XMMRegister xtmp2, Register rtmp, int vec_enc);
413 
414   void vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
415                                XMMRegister xtmp, Register rscratch = noreg);
416 
417   void vector_reverse_byte(BasicType bt, XMMRegister dst, XMMRegister src, int vec_enc);
418 
419   void vector_popcount_int(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
420                            XMMRegister xtmp2, Register rtmp, int vec_enc);
421 
422   void vector_popcount_long(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
423                             XMMRegister xtmp2, Register rtmp, int vec_enc);
424 
425   void vector_popcount_short(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
426                              XMMRegister xtmp2, Register rtmp, int vec_enc);
427 
428   void vector_popcount_byte(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
429                             XMMRegister xtmp2, Register rtmp, int vec_enc);
430 
431   void vector_popcount_integral(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
432                                 XMMRegister xtmp2, Register rtmp, int vec_enc);
433 
434   void vector_popcount_integral_evex(BasicType bt, XMMRegister dst, XMMRegister src,
435                                      KRegister mask, bool merge, int vec_enc);
436 
437   void vbroadcast(BasicType bt, XMMRegister dst, int imm32, Register rtmp, int vec_enc);
438 
439   void vector_reverse_byte64(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
440                              XMMRegister xtmp2, Register rtmp, int vec_enc);
441 
442   void vector_count_leading_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src,
443                                        XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
444                                        KRegister ktmp, Register rtmp, bool merge, int vec_enc);
445 
446   void vector_count_leading_zeros_byte_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
447                                            XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
448 
449   void vector_count_leading_zeros_short_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
450                                             XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
451 
452   void vector_count_leading_zeros_int_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
453                                           XMMRegister xtmp2, XMMRegister xtmp3, int vec_enc);
454 
455   void vector_count_leading_zeros_long_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
456                                            XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
457 
458   void vector_count_leading_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
459                                       XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
460 
461   void vpadd(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
462 
463   void vpsub(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc);
464 
465   void vector_count_trailing_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
466                                         XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, KRegister ktmp,
467                                         Register rtmp, int vec_enc);
468 
469   void vector_swap_nbits(int nbits, int bitmask, XMMRegister dst, XMMRegister src,
470                          XMMRegister xtmp1, Register rtmp, int vec_enc);
471 
472   void vector_count_trailing_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
473                                        XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc);
474 
475   void vector_signum_avx(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
476                          XMMRegister xtmp1, int vec_enc);
477 
478   void vector_signum_evex(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
479                           KRegister ktmp1, int vec_enc);
480 
481   void vmovmask(BasicType elem_bt, XMMRegister dst, Address src, XMMRegister mask, int vec_enc);
482 
483   void vmovmask(BasicType elem_bt, Address dst, XMMRegister src, XMMRegister mask, int vec_enc);
484 
485   void rearrange_bytes(XMMRegister dst, XMMRegister shuffle, XMMRegister src, XMMRegister xtmp1,
486                        XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, KRegister ktmp, int vlen_enc);
487 
488   void vector_rearrange_int_float(BasicType bt, XMMRegister dst, XMMRegister shuffle,
489                                   XMMRegister src, int vlen_enc);
490 
491   void efp16sh(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
492 
493   void vgather_subword(BasicType elem_ty, XMMRegister dst,  Register base, Register idx_base, Register offset,
494                        Register mask, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp,
495                        Register midx, Register length, int vector_len, int vlen_enc);
496 
497   void vgather8b_masked_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
498                                Register offset, Register mask, Register midx, Register rtmp, int vlen_enc);
499 
500   void vgather8b_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
501                               Register offset, Register rtmp, int vlen_enc);
502 
503   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, bool is_unsigned, int vlen_enc);
504 
505   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, bool is_unsigned, int vlen_enc);
506 
507   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
508 
509   void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
510 
511   void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
512 
513   void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
514 
515   void vector_sub_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, KRegister ktmp, int vlen_enc);
516 
517   void vector_sub_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
518                                              XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
519 
520   void vector_add_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
521                                               XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp, int vlen_enc);
522 
523   void vector_add_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
524                                              XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, int vlen_enc);
525 
526   void vector_addsub_dq_saturating_avx(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
527                                        XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, int vlen_enc);
528 
529   void vector_addsub_dq_saturating_evex(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
530                                         XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, int vlen_enc);
531 
532   void evpmovd2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
533 
534   void evpmovq2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
535 
536   void vpsign_extend_dq(BasicType etype, XMMRegister dst, XMMRegister src, int vlen_enc);
537 
538   void vpgenmin_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
539 
540   void vpgenmax_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
541 
542   void evpcmpu(BasicType etype, KRegister kmask,  XMMRegister src1, XMMRegister src2, Assembler::ComparisonPredicate cond, int vlen_enc);
543 
544   void vpcmpgt(BasicType etype, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
545 
546   void evpmov_vec_to_mask(BasicType etype, KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
547                           int vlen_enc, bool xtmp2_hold_M1 = false);
548 
549   void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
550                               bool is_unsigned, bool merge, int vlen_enc);
551 
552   void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
553                               bool is_unsigned, bool merge, int vlen_enc);
554 
555   void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
556                               bool merge, int vlen_enc);
557 
558   void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
559                               bool merge, int vlen_enc);
560 
561   void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
562                                        XMMRegister src2, bool merge, int vlen_enc);
563 
564   void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
565                                        Address src2, bool merge, int vlen_enc);
566 
567   void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
568 
569   void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
570 
571   void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
572 
573   void vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
574                           KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
575 
576   void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
577                           KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
578 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP