1 /*
2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef CPU_X86_C2_MACROASSEMBLER_X86_HPP
26 #define CPU_X86_C2_MACROASSEMBLER_X86_HPP
27
28 // C2_MacroAssembler contains high-level macros for C2
29
30 public:
31 Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);
32
33 // special instructions for EVEX
34 void setvectmask(Register dst, Register src, KRegister mask);
35 void restorevectmask(KRegister mask);
36
37 // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
38 // See full desription in macroAssembler_x86.cpp.
39 void fast_lock(Register obj, Register box, Register tmp,
40 Register scr, Register cx1, Register cx2,
41 RTMLockingCounters* rtm_counters,
42 RTMLockingCounters* stack_rtm_counters,
43 Metadata* method_data,
44 bool use_rtm, bool profile_rtm);
45 void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
46
47 #if INCLUDE_RTM_OPT
48 void rtm_counters_update(Register abort_status, Register rtm_counters);
49 void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
50 void rtm_abort_ratio_calculation(Register tmp, Register rtm_counters_reg,
51 RTMLockingCounters* rtm_counters,
52 Metadata* method_data);
53 void rtm_profiling(Register abort_status_Reg, Register rtm_counters_Reg,
54 RTMLockingCounters* rtm_counters, Metadata* method_data, bool profile_rtm);
55 void rtm_retry_lock_on_abort(Register retry_count, Register abort_status, Label& retryLabel);
56 void rtm_retry_lock_on_busy(Register retry_count, Register box, Register tmp, Register scr, Label& retryLabel);
57 void rtm_stack_locking(Register obj, Register tmp, Register scr,
58 Register retry_on_abort_count,
59 RTMLockingCounters* stack_rtm_counters,
60 Metadata* method_data, bool profile_rtm,
61 Label& DONE_LABEL, Label& IsInflated);
62 void rtm_inflated_locking(Register obj, Register box, Register tmp,
63 Register scr, Register retry_on_busy_count,
64 Register retry_on_abort_count,
65 RTMLockingCounters* rtm_counters,
66 Metadata* method_data, bool profile_rtm,
67 Label& DONE_LABEL);
68 #endif
69
70 // Generic instructions support for use in .ad files C2 code generation
71 void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, Register scr);
72 void vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr);
73 void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, Register scr);
74 void vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr);
75
76 void pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src,
77 XMMRegister tmp = xnoreg);
78 void vpminmax(int opcode, BasicType elem_bt,
79 XMMRegister dst, XMMRegister src1, XMMRegister src2,
80 int vlen_enc);
81
82 void vminmax_fp(int opcode, BasicType elem_bt,
83 XMMRegister dst, XMMRegister a, XMMRegister b,
84 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
85 int vlen_enc);
86 void evminmax_fp(int opcode, BasicType elem_bt,
87 XMMRegister dst, XMMRegister a, XMMRegister b,
88 KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
89 int vlen_enc);
90
91 void signum_fp(int opcode, XMMRegister dst,
92 XMMRegister zero, XMMRegister one,
93 Register scratch);
94
95 void vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
96 void vextendbw(bool sign, XMMRegister dst, XMMRegister src);
97 void vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
98 void vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len);
99
100 void vshiftd(int opcode, XMMRegister dst, XMMRegister shift);
101 void vshiftd_imm(int opcode, XMMRegister dst, int shift);
102 void vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
103 void vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
104 void vshiftw(int opcode, XMMRegister dst, XMMRegister shift);
105 void vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
106 void vshiftq(int opcode, XMMRegister dst, XMMRegister shift);
107 void vshiftq_imm(int opcode, XMMRegister dst, int shift);
108 void vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
109 void vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len);
110
111 void vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, int shift, int vector_len);
112 void vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
113
114 void varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
115 void varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc);
116 void varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister vtmp = xnoreg);
117 void varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp, Register scratch);
118 void evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp, Register scratch);
119
120 void insert(BasicType typ, XMMRegister dst, Register val, int idx);
121 void vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx);
122 void vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len);
123 void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
124 void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
125
126 void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len);
127 void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len);
128
129 // extract
130 void extract(BasicType typ, Register dst, XMMRegister src, int idx);
131 XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
132 void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex);
133 void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, Register tmp = noreg, XMMRegister vtmp = xnoreg);
134
135 // vector test
136 void vectortest(int bt, int vlen, XMMRegister src1, XMMRegister src2,
137 XMMRegister vtmp1 = xnoreg, XMMRegister vtmp2 = xnoreg, KRegister mask = knoreg);
138
139 // blend
140 void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register scratch = rscratch1);
141 void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
142 void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
143
144 void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
145 void load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, Register tmp, bool novlbwdq, int vlen_enc);
146
147 void load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes);
148
149 // vector compare
150 void vpcmpu(BasicType typ, XMMRegister dst, XMMRegister src1, XMMRegister src2, ComparisonPredicate comparison, int vlen_in_bytes,
151 XMMRegister vtmp1, XMMRegister vtmp2, Register scratch);
152 void vpcmpu32(BasicType typ, XMMRegister dst, XMMRegister src1, XMMRegister src2, ComparisonPredicate comparison, int vlen_in_bytes,
153 XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3, Register scratch);
154
155 // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
156
157 // dst = src1 reduce(op, src2) using vtmp as temps
158 void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
159 #ifdef _LP64
160 void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
161 void genmask(KRegister dst, Register len, Register temp);
162 #endif // _LP64
163
164 // dst = reduce(op, src2) using vtmp as temps
165 void reduce_fp(int opcode, int vlen,
166 XMMRegister dst, XMMRegister src,
167 XMMRegister vtmp1, XMMRegister vtmp2 = xnoreg);
168 void reduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
169 void mulreduceB(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
170 void reduceS(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
171 void reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
172 XMMRegister dst, XMMRegister src,
173 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
174 void reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid,
175 XMMRegister dst, XMMRegister src,
176 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp, XMMRegister xmm_0, XMMRegister xmm_1 = xnoreg);
177 private:
178 void reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
179 void reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
180
181 // Int Reduction
182 void reduce2I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
183 void reduce4I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
184 void reduce8I (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
185 void reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
186
187 // Byte Reduction
188 void reduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
189 void reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
190 void reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
191 void reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
192 void mulreduce8B (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
193 void mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
194 void mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
195 void mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
196
197 // Short Reduction
198 void reduce4S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
199 void reduce8S (int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
200 void reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
201 void reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
202
203 // Long Reduction
204 #ifdef _LP64
205 void reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
206 void reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
207 void reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
208 #endif // _LP64
209
210 // Float Reduction
211 void reduce2F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
212 void reduce4F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
213 void reduce8F (int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
214 void reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
215
216 // Double Reduction
217 void reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp);
218 void reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
219 void reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2);
220
221 // Base reduction instruction
222 void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src);
223 void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2);
224
225 public:
226 #ifdef _LP64
227 void vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp, int masklen, int masksize, int vec_enc);
228
229 void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, XMMRegister xtmp1,
230 Register tmp, int masklen, int masksize, int vec_enc);
231 #endif
232 void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
233 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
234
235 void stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
236 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
237
238 // IndexOf strings.
239 // Small strings are loaded through stack if they cross page boundary.
240 void string_indexof(Register str1, Register str2,
241 Register cnt1, Register cnt2,
242 int int_cnt2, Register result,
243 XMMRegister vec, Register tmp,
244 int ae);
245
246 // IndexOf for constant substrings with size >= 8 elements
247 // which don't need to be loaded through stack.
248 void string_indexofC8(Register str1, Register str2,
249 Register cnt1, Register cnt2,
250 int int_cnt2, Register result,
251 XMMRegister vec, Register tmp,
252 int ae);
253
254 // Smallest code: we don't need to load through stack,
255 // check string tail.
256
257 // helper function for string_compare
258 void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
259 Address::ScaleFactor scale, Address::ScaleFactor scale1,
260 Address::ScaleFactor scale2, Register index, int ae);
261 // Compare strings.
262 void string_compare(Register str1, Register str2,
263 Register cnt1, Register cnt2, Register result,
264 XMMRegister vec1, int ae, KRegister mask = knoreg);
265
266 // Search for Non-ASCII character (Negative byte value) in a byte array,
267 // return true if it has any and false otherwise.
268 void has_negatives(Register ary1, Register len,
269 Register result, Register tmp1,
270 XMMRegister vec1, XMMRegister vec2, KRegister mask1 = knoreg, KRegister mask2 = knoreg);
271
272 // Compare char[] or byte[] arrays.
273 void arrays_equals(bool is_array_equ, Register ary1, Register ary2,
274 Register limit, Register result, Register chr,
275 XMMRegister vec1, XMMRegister vec2, bool is_char, KRegister mask = knoreg);
276
277
278 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
279 XMMRegister dst, XMMRegister src1, XMMRegister src2,
280 bool merge, int vlen_enc, bool is_varshift = false);
281
282 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask,
283 XMMRegister dst, XMMRegister src1, Address src2,
284 bool merge, int vlen_enc);
285
286 void evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
287 XMMRegister src1, int imm8, bool merge, int vlen_enc);
288
289 void masked_op(int ideal_opc, int mask_len, KRegister dst,
290 KRegister src1, KRegister src2);
291 #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP