1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef CPU_X86_C2_MACROASSEMBLER_X86_HPP
 26 #define CPU_X86_C2_MACROASSEMBLER_X86_HPP
 27 
 28 // C2_MacroAssembler contains high-level macros for C2
 29 
 30 public:
 31   Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);
 32 
 33   // special instructions for EVEX
 34   void setvectmask(Register dst, Register src, KRegister mask);
 35   void restorevectmask(KRegister mask);
 36 
 37   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
 38   // See full desription in macroAssembler_x86.cpp.
 39   void fast_lock(Register obj, Register box, Register tmp,
 40                  Register scr, Register cx1, Register cx2, Register thread,
 41                  BiasedLockingCounters* counters,
 42                  RTMLockingCounters* rtm_counters,
 43                  RTMLockingCounters* stack_rtm_counters,
 44                  Metadata* method_data,
 45                  bool use_rtm, bool profile_rtm);
 46   void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
 47 
 48 #if INCLUDE_RTM_OPT
 49   void rtm_counters_update(Register abort_status, Register rtm_counters);
 50   void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
 51   void rtm_abort_ratio_calculation(Register tmp, Register rtm_counters_reg,
 52                                    RTMLockingCounters* rtm_counters,
 53                                    Metadata* method_data);
 54   void rtm_profiling(Register abort_status_Reg, Register rtm_counters_Reg,
 55                      RTMLockingCounters* rtm_counters, Metadata* method_data, bool profile_rtm);
 56   void rtm_retry_lock_on_abort(Register retry_count, Register abort_status, Label& retryLabel);
 57   void rtm_retry_lock_on_busy(Register retry_count, Register box, Register tmp, Register scr, Label& retryLabel);
 58   void rtm_stack_locking(Register obj, Register tmp, Register scr,
 59                          Register retry_on_abort_count,
 60                          RTMLockingCounters* stack_rtm_counters,
 61                          Metadata* method_data, bool profile_rtm,
 62                          Label& DONE_LABEL, Label& IsInflated);
 63   void rtm_inflated_locking(Register obj, Register box, Register tmp,
 64                             Register scr, Register retry_on_busy_count,
 65                             Register retry_on_abort_count,
 66                             RTMLockingCounters* rtm_counters,
 67                             Metadata* method_data, bool profile_rtm,
 68                             Label& DONE_LABEL);
 69 #endif
 70 
 71   // Generic instructions support for use in .ad files C2 code generation
 72   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, Register scr);
 73   void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr);
 74   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, Register scr);
 75   void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr);
 76 
 77   void pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src,
 78                XMMRegister tmp = xnoreg);
 79   void vpminmax(int opcode, BasicType elem_bt,
 80                 XMMRegister dst, XMMRegister src1, XMMRegister src2,
 81                 int vlen_enc);
 82 
 83   void vminmax_fp(int opcode, BasicType elem_bt,
 84                   XMMRegister dst, XMMRegister a, XMMRegister b,
 85                   XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
 86                   int vlen_enc);
 87   void evminmax_fp(int opcode, BasicType elem_bt,
 88                    XMMRegister dst, XMMRegister a, XMMRegister b,
 89                    KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
 90                    int vlen_enc);
 91 
 92   void signum_fp(int opcode, XMMRegister dst,
 93                  XMMRegister zero, XMMRegister one,
 94                  Register scratch);
 95 
 96   void vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 97   void vextendbw(bool sign, XMMRegister dst, XMMRegister src);
 98   void vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
 99   void vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
100 
101   void vshiftd(int opcode, XMMRegister dst, XMMRegister shift);
102   void vshiftd_imm(int opcode, XMMRegister dst, int shift);
103   void vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
104   void vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
105   void vshiftw(int opcode, XMMRegister dst, XMMRegister shift);
106   void vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
107   void vshiftq(int opcode, XMMRegister dst, XMMRegister shift);
108   void vshiftq_imm(int opcode, XMMRegister dst, int shift);
109   void vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
110   void vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
111 
112   void vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, int shift, int vector_len);
113   void vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
114 
115   void varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
116   void varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
117   void varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister vtmp = xnoreg);
118   void varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp, Register scratch);
119   void evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp, Register scratch);
120 
121   void insert(BasicType typ, XMMRegister dst, Register val, int idx);
122   void vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx);
123   void vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len);
124   void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
125   void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
126 
127   void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len);
128   void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len);
129 
130   // extract
131   void extract(BasicType typ, Register dst, XMMRegister src, int idx);
132   XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
133   void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex);
134   void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, Register tmp = noreg, XMMRegister vtmp = xnoreg);
135 
136   // vector test
137   void vectortest(int bt, int vlen, XMMRegister src1, XMMRegister src2,
138                   XMMRegister vtmp1 = xnoreg, XMMRegister vtmp2 = xnoreg, KRegister mask = knoreg);
139 
140   // blend
141   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register scratch = rscratch1);
142   void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
143   void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
144 
145   void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
146   void load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes);
147 
148   // vector compare
149   void vpcmpu(BasicType typ, XMMRegister dst, XMMRegister src1, XMMRegister src2, ComparisonPredicate comparison, int vlen_in_bytes,
150               XMMRegister vtmp1, XMMRegister vtmp2, Register scratch);
151   void vpcmpu32(BasicType typ, XMMRegister dst, XMMRegister src1, XMMRegister src2, ComparisonPredicate comparison, int vlen_in_bytes,
152                 XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3, Register scratch);
153 
154   // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
155 
156   // dst = src1  reduce(op, src2) using vtmp as temps
157   void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
158 #ifdef _LP64
159   void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
160   void genmask(KRegister dst, Register len, Register temp);
161 #endif // _LP64
162 
163   // dst = reduce(op, src2) using vtmp as temps
164   void reduce_fp(int opcode, int vlen,
165                  XMMRegister dst, XMMRegister src,
166                  XMMRegister vtmp1, XMMRegister vtmp2 = xnoreg);
167   void reduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
168   void mulreduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
169   void reduceS(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
170   void reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
171                          XMMRegister dst, XMMRegister src,
172                          XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
173   void reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid,
174                           XMMRegister dst, XMMRegister src,
175                           XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
176  private:
177   void reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
178   void reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
179 
180   // Int Reduction
181   void reduce2I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
182   void reduce4I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
183   void reduce8I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
184   void reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
185 
186   // Byte Reduction
187   void reduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
188   void reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
189   void reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
190   void reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
191   void mulreduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
192   void mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
193   void mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
194   void mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
195 
196   // Short Reduction
197   void reduce4S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
198   void reduce8S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
199   void reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
200   void reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
201 
202   // Long Reduction
203 #ifdef _LP64
204   void reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
205   void reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
206   void reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
207 #endif // _LP64
208 
209   // Float Reduction
210   void reduce2F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
211   void reduce4F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
212   void reduce8F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
213   void reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
214 
215   // Double Reduction
216   void reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
217   void reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
218   void reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
219 
220   // Base reduction instruction
221   void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
222   void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
223 
224  public:
225 #ifdef _LP64
226   void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, Register tmp,
227                              KRegister ktmp, int masklen, int vec_enc);
228 
229   void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, XMMRegister xtmp1,
230                              Register tmp, int masklen, int vec_enc);
231 #endif
232   void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
233                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
234 
235   void stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
236                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
237 
238   // IndexOf strings.
239   // Small strings are loaded through stack if they cross page boundary.
240   void string_indexof(Register str1, Register str2,
241                       Register cnt1, Register cnt2,
242                       int int_cnt2,  Register result,
243                       XMMRegister vec, Register tmp,
244                       int ae);
245 
246   // IndexOf for constant substrings with size >= 8 elements
247   // which don't need to be loaded through stack.
248   void string_indexofC8(Register str1, Register str2,
249                       Register cnt1, Register cnt2,
250                       int int_cnt2,  Register result,
251                       XMMRegister vec, Register tmp,
252                       int ae);
253 
254     // Smallest code: we don't need to load through stack,
255     // check string tail.
256 
257   // helper function for string_compare
258   void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
259                           Address::ScaleFactor scale, Address::ScaleFactor scale1,
260                           Address::ScaleFactor scale2, Register index, int ae);
261   // Compare strings.
262   void string_compare(Register str1, Register str2,
263                       Register cnt1, Register cnt2, Register result,
264                       XMMRegister vec1, int ae, KRegister mask = knoreg);
265 
266   // Search for Non-ASCII character (Negative byte value) in a byte array,
267   // return true if it has any and false otherwise.
268   void has_negatives(Register ary1, Register len,
269                      Register result, Register tmp1,
270                      XMMRegister vec1, XMMRegister vec2, KRegister mask1 = knoreg, KRegister mask2 = knoreg);
271 
272   // Compare char[] or byte[] arrays.
273   void arrays_equals(bool is_array_equ, Register ary1, Register ary2,
274                      Register limit, Register result, Register chr,
275                      XMMRegister vec1, XMMRegister vec2, bool is_char, KRegister mask = knoreg);
276 
277   void rearrange_bytes(XMMRegister dst, XMMRegister shuffle, XMMRegister src, XMMRegister xtmp1,
278                        XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, KRegister ktmp, int vlen_enc);
279 
280 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP