1 //
2 // Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
223 // Word a in each register holds a Float, words ab hold a Double.
224 // The whole registers are used in SSE4.2 version intrinsics,
225 // array copy stubs and superword operations (see UseSSE42Intrinsics,
226 // UseXMMForArrayCopy and UseSuperword flags).
227 // For pre EVEX enabled architectures:
228 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
229 // For EVEX enabled architectures:
230 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
231 //
232 // Linux ABI: No register preserved across function calls
233 // XMM0-XMM7 might hold parameters
234 // Windows ABI: XMM6-XMM15 preserved across function calls
235 // XMM0-XMM3 might hold parameters
236
237 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
238 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
239 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
240 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
241 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
242 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
243 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
244 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
245 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
246 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
247 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
248 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
249 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
250 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
251 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
252 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
253
254 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
255 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
256 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
257 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
258 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
259 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
260 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
261 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
262 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
263 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
264 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
265 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
266 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
267 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
268 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
269 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
270
271 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
272 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
273 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
274 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
275 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
276 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
277 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
278 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
279 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
280 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
281 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
282 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
283 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
284 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
285 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
286 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
287
288 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
289 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
290 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
291 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
292 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
293 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
294 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
295 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
296 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
297 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
298 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
299 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
300 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
301 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
302 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
303 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
304
305 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
306 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
307 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
308 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
309 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
310 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
311 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
312 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
313 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
314 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
315 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
316 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
317 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
318 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
319 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
320 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
321
322 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
323 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
324 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
325 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
326 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
327 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
328 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
329 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
330 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
331 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
332 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
333 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
334 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
335 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
336 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
337 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
338
339 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
340 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
341 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
342 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
343 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
344 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
345 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
346 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
347 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
348 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
349 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
350 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
351 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
352 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
353 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
354 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
355
356 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
357 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
358 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
359 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
360 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
361 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
362 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
363 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
364 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
365 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
366 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
367 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
368 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
369 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
370 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
371 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
372
373 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
374 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
375 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
376 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
377 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
378 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
379 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
380 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
381 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
382 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
383 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
384 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
385 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
386 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
387 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
388 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
389
390 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
391 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
392 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
393 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
394 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
395 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
396 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
397 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
398 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
399 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
400 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
401 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
402 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
403 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
404 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
405 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
406
407 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
408 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
409 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
410 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
411 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
412 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
413 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
414 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
415 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
416 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
417 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
418 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
419 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
420 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
421 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
422 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
423
424 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
425 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
426 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
427 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
428 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
429 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
430 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
431 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
432 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
433 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
434 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
435 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
436 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
437 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
438 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
439 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
440
441 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
442 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
443 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
444 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
445 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
446 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
447 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
448 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
449 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
450 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
451 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
452 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
453 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
454 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
455 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
456 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
457
458 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
459 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
460 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
461 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
462 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
463 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
464 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
465 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
466 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
467 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
468 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
469 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
470 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
471 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
472 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
473 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
474
475 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
476 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
477 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
478 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
479 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
480 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
481 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
482 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
483 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
484 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
485 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
486 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
487 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
488 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
489 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
490 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
491
492 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
493 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
494 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
495 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
496 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
497 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
498 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
499 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
500 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
501 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
502 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
503 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
504 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
505 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
506 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
507 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
508
509 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
510 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
511 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
512 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
513 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
514 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
515 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
516 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
517 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
518 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
519 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
520 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
521 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
522 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
523 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
524 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
525
526 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
527 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
528 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
529 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
530 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
531 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
532 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
533 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
534 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
535 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
536 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
537 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
538 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
539 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
540 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
541 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
542
543 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
544 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
545 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
546 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
547 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
548 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
549 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
550 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
551 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
552 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
553 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
554 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
555 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
556 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
557 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
558 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
559
560 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
561 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
562 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
563 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
564 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
565 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
566 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
567 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
568 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
569 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
570 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
571 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
572 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
573 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
574 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
575 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
576
577 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
578 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
579 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
580 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
581 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
582 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
583 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
584 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
585 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
586 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
587 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
588 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
589 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
590 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
591 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
592 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
593
594 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
595 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
596 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
597 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
598 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
599 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
600 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
601 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
602 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
603 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
604 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
605 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
606 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
607 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
608 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
609 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
610
611 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
612 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
613 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
614 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
615 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
616 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
617 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
618 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
619 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
620 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
621 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
622 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
623 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
624 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
625 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
626 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
627
628 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
629 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
630 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
631 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
632 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
633 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
634 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
635 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
636 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
637 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
638 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
639 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
640 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
641 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
642 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
643 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
644
645 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
646 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
647 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
648 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
649 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
650 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
651 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
652 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
653 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
654 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
655 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
656 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
657 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
658 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
659 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
660 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
661
662 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
663 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
664 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
665 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
666 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
667 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
668 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
669 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
670 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
671 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
672 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
673 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
674 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
675 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
676 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
677 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
678
679 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
680 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
681 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
682 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
683 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
684 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
685 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
686 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
687 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
688 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
689 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
690 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
691 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
692 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
693 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
694 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
695
696 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
697 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
698 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
699 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
700 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
701 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
702 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
703 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
704 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
705 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
706 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
707 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
708 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
709 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
710 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
711 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
712
713 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
714 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
715 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
716 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
717 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
718 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
719 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
720 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
721 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
722 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
723 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
724 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
725 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
726 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
727 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
728 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
729
730 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
731 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
732 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
733 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
734 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
735 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
736 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
737 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
738 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
739 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
740 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
741 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
742 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
743 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
744 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
745 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
746
747 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
748 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
749 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
750 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
751 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
752 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
753 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
754 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
755 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
756 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
757 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
758 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
759 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
760 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
761 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
762 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
763
764 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
765 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
766 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
767 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
768 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
769 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
770 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
771 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
772 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
773 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
774 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
775 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
776 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
777 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
778 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
779 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
780
781 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
782
783 // AVX3 Mask Registers.
784 reg_def K1 (SOC, SOC, Op_RegI, 1, k1->as_VMReg());
785 reg_def K1_H (SOC, SOC, Op_RegI, 1, k1->as_VMReg()->next());
786
787 reg_def K2 (SOC, SOC, Op_RegI, 2, k2->as_VMReg());
788 reg_def K2_H (SOC, SOC, Op_RegI, 2, k2->as_VMReg()->next());
789
790 reg_def K3 (SOC, SOC, Op_RegI, 3, k3->as_VMReg());
791 reg_def K3_H (SOC, SOC, Op_RegI, 3, k3->as_VMReg()->next());
792
793 reg_def K4 (SOC, SOC, Op_RegI, 4, k4->as_VMReg());
794 reg_def K4_H (SOC, SOC, Op_RegI, 4, k4->as_VMReg()->next());
795
796 reg_def K5 (SOC, SOC, Op_RegI, 5, k5->as_VMReg());
797 reg_def K5_H (SOC, SOC, Op_RegI, 5, k5->as_VMReg()->next());
798
799 reg_def K6 (SOC, SOC, Op_RegI, 6, k6->as_VMReg());
800 reg_def K6_H (SOC, SOC, Op_RegI, 6, k6->as_VMReg()->next());
801
802 reg_def K7 (SOC, SOC, Op_RegI, 7, k7->as_VMReg());
803 reg_def K7_H (SOC, SOC, Op_RegI, 7, k7->as_VMReg()->next());
804
805
806 //----------Architecture Description Register Classes--------------------------
807 // Several register classes are automatically defined based upon information in
808 // this architecture description.
809 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
810 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
811 //
812
813 // Empty register class.
814 reg_class no_reg();
815
816 // Class for all pointer/long registers including APX extended GPRs.
817 reg_class all_reg(RAX, RAX_H,
818 RDX, RDX_H,
819 RBP, RBP_H,
820 RDI, RDI_H,
821 RSI, RSI_H,
822 RCX, RCX_H,
823 RBX, RBX_H,
824 RSP, RSP_H,
825 R8, R8_H,
826 R9, R9_H,
827 R10, R10_H,
828 R11, R11_H,
829 R12, R12_H,
830 R13, R13_H,
831 R14, R14_H,
832 R15, R15_H,
833 R16, R16_H,
834 R17, R17_H,
835 R18, R18_H,
836 R19, R19_H,
837 R20, R20_H,
838 R21, R21_H,
839 R22, R22_H,
840 R23, R23_H,
841 R24, R24_H,
842 R25, R25_H,
843 R26, R26_H,
844 R27, R27_H,
845 R28, R28_H,
846 R29, R29_H,
847 R30, R30_H,
848 R31, R31_H);
849
850 // Class for all int registers including APX extended GPRs.
851 reg_class all_int_reg(RAX
852 RDX,
853 RBP,
854 RDI,
855 RSI,
856 RCX,
857 RBX,
858 R8,
859 R9,
860 R10,
861 R11,
862 R12,
863 R13,
864 R14,
865 R16,
866 R17,
867 R18,
868 R19,
869 R20,
870 R21,
871 R22,
872 R23,
873 R24,
874 R25,
875 R26,
876 R27,
877 R28,
878 R29,
879 R30,
880 R31);
881
882 // Class for all pointer registers
883 reg_class any_reg %{
884 return _ANY_REG_mask;
885 %}
886
887 // Class for all pointer registers (excluding RSP)
888 reg_class ptr_reg %{
889 return _PTR_REG_mask;
890 %}
891
892 // Class for all pointer registers (excluding RSP and RBP)
893 reg_class ptr_reg_no_rbp %{
894 return _PTR_REG_NO_RBP_mask;
895 %}
896
897 // Class for all pointer registers (excluding RAX and RSP)
898 reg_class ptr_no_rax_reg %{
899 return _PTR_NO_RAX_REG_mask;
900 %}
901
902 // Class for all pointer registers (excluding RAX, RBX, and RSP)
903 reg_class ptr_no_rax_rbx_reg %{
904 return _PTR_NO_RAX_RBX_REG_mask;
905 %}
906
907 // Class for all long registers (excluding RSP)
908 reg_class long_reg %{
909 return _LONG_REG_mask;
910 %}
911
912 // Class for all long registers (excluding RAX, RDX and RSP)
913 reg_class long_no_rax_rdx_reg %{
914 return _LONG_NO_RAX_RDX_REG_mask;
915 %}
916
917 // Class for all long registers (excluding RCX and RSP)
918 reg_class long_no_rcx_reg %{
919 return _LONG_NO_RCX_REG_mask;
920 %}
921
922 // Class for all long registers (excluding RBP and R13)
923 reg_class long_no_rbp_r13_reg %{
924 return _LONG_NO_RBP_R13_REG_mask;
925 %}
926
927 // Class for all int registers (excluding RSP)
928 reg_class int_reg %{
929 return _INT_REG_mask;
930 %}
931
932 // Class for all int registers (excluding RAX, RDX, and RSP)
933 reg_class int_no_rax_rdx_reg %{
934 return _INT_NO_RAX_RDX_REG_mask;
935 %}
936
937 // Class for all int registers (excluding RCX and RSP)
938 reg_class int_no_rcx_reg %{
939 return _INT_NO_RCX_REG_mask;
940 %}
941
942 // Class for all int registers (excluding RBP and R13)
943 reg_class int_no_rbp_r13_reg %{
944 return _INT_NO_RBP_R13_REG_mask;
945 %}
946
947 // Singleton class for RAX pointer register
948 reg_class ptr_rax_reg(RAX, RAX_H);
949
950 // Singleton class for RBX pointer register
951 reg_class ptr_rbx_reg(RBX, RBX_H);
952
953 // Singleton class for RSI pointer register
954 reg_class ptr_rsi_reg(RSI, RSI_H);
955
956 // Singleton class for RBP pointer register
957 reg_class ptr_rbp_reg(RBP, RBP_H);
958
959 // Singleton class for RDI pointer register
960 reg_class ptr_rdi_reg(RDI, RDI_H);
961
962 // Singleton class for stack pointer
963 reg_class ptr_rsp_reg(RSP, RSP_H);
964
965 // Singleton class for TLS pointer
966 reg_class ptr_r15_reg(R15, R15_H);
967
968 // Singleton class for RAX long register
969 reg_class long_rax_reg(RAX, RAX_H);
970
971 // Singleton class for RCX long register
972 reg_class long_rcx_reg(RCX, RCX_H);
973
974 // Singleton class for RDX long register
975 reg_class long_rdx_reg(RDX, RDX_H);
976
977 // Singleton class for R11 long register
978 reg_class long_r11_reg(R11, R11_H);
979
980 // Singleton class for RAX int register
981 reg_class int_rax_reg(RAX);
982
983 // Singleton class for RBX int register
984 reg_class int_rbx_reg(RBX);
985
986 // Singleton class for RCX int register
987 reg_class int_rcx_reg(RCX);
988
989 // Singleton class for RDX int register
990 reg_class int_rdx_reg(RDX);
991
992 // Singleton class for RDI int register
993 reg_class int_rdi_reg(RDI);
994
995 // Singleton class for instruction pointer
996 // reg_class ip_reg(RIP);
997
998 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
999 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1000 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1001 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1002 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1003 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1004 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1005 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1006 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1007 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1008 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1009 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1010 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1011 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1012 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1013 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1014 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1015 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1016 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1017 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1018 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1019 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1020 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1021 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1022 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1023 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1024 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1025 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1026 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1027 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1028 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1029 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1030
1031 alloc_class chunk2(K7, K7_H,
1032 K6, K6_H,
1033 K5, K5_H,
1034 K4, K4_H,
1035 K3, K3_H,
1036 K2, K2_H,
1037 K1, K1_H);
1038
1039 reg_class vectmask_reg(K1, K1_H,
1040 K2, K2_H,
1041 K3, K3_H,
1042 K4, K4_H,
1043 K5, K5_H,
1044 K6, K6_H,
1045 K7, K7_H);
1046
1047 reg_class vectmask_reg_K1(K1, K1_H);
1048 reg_class vectmask_reg_K2(K2, K2_H);
1049 reg_class vectmask_reg_K3(K3, K3_H);
1050 reg_class vectmask_reg_K4(K4, K4_H);
1051 reg_class vectmask_reg_K5(K5, K5_H);
1052 reg_class vectmask_reg_K6(K6, K6_H);
1053 reg_class vectmask_reg_K7(K7, K7_H);
1054
1055 // flags allocation class should be last.
1056 alloc_class chunk3(RFLAGS);
1057
1058 // Singleton class for condition codes
1059 reg_class int_flags(RFLAGS);
1060
1061 // Class for pre evex float registers
1062 reg_class float_reg_legacy(XMM0,
1063 XMM1,
1064 XMM2,
1065 XMM3,
1066 XMM4,
1067 XMM5,
1068 XMM6,
1069 XMM7,
1070 XMM8,
1071 XMM9,
1072 XMM10,
1073 XMM11,
1074 XMM12,
1075 XMM13,
1076 XMM14,
1077 XMM15);
1078
1079 // Class for evex float registers
1080 reg_class float_reg_evex(XMM0,
1081 XMM1,
1082 XMM2,
1083 XMM3,
1084 XMM4,
1085 XMM5,
1086 XMM6,
1087 XMM7,
1088 XMM8,
1089 XMM9,
1090 XMM10,
1091 XMM11,
1092 XMM12,
1093 XMM13,
1094 XMM14,
1095 XMM15,
1096 XMM16,
1097 XMM17,
1098 XMM18,
1099 XMM19,
1100 XMM20,
1101 XMM21,
1102 XMM22,
1103 XMM23,
1104 XMM24,
1105 XMM25,
1106 XMM26,
1107 XMM27,
1108 XMM28,
1109 XMM29,
1110 XMM30,
1111 XMM31);
1112
1113 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1114 reg_class_dynamic float_reg_vl(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1115
1116 // Class for pre evex double registers
1117 reg_class double_reg_legacy(XMM0, XMM0b,
1118 XMM1, XMM1b,
1119 XMM2, XMM2b,
1120 XMM3, XMM3b,
1121 XMM4, XMM4b,
1122 XMM5, XMM5b,
1123 XMM6, XMM6b,
1124 XMM7, XMM7b,
1125 XMM8, XMM8b,
1126 XMM9, XMM9b,
1127 XMM10, XMM10b,
1128 XMM11, XMM11b,
1129 XMM12, XMM12b,
1130 XMM13, XMM13b,
1131 XMM14, XMM14b,
1132 XMM15, XMM15b);
1133
1134 // Class for evex double registers
1135 reg_class double_reg_evex(XMM0, XMM0b,
1136 XMM1, XMM1b,
1137 XMM2, XMM2b,
1138 XMM3, XMM3b,
1139 XMM4, XMM4b,
1140 XMM5, XMM5b,
1141 XMM6, XMM6b,
1142 XMM7, XMM7b,
1143 XMM8, XMM8b,
1144 XMM9, XMM9b,
1145 XMM10, XMM10b,
1146 XMM11, XMM11b,
1147 XMM12, XMM12b,
1148 XMM13, XMM13b,
1149 XMM14, XMM14b,
1150 XMM15, XMM15b,
1151 XMM16, XMM16b,
1152 XMM17, XMM17b,
1153 XMM18, XMM18b,
1154 XMM19, XMM19b,
1155 XMM20, XMM20b,
1156 XMM21, XMM21b,
1157 XMM22, XMM22b,
1158 XMM23, XMM23b,
1159 XMM24, XMM24b,
1160 XMM25, XMM25b,
1161 XMM26, XMM26b,
1162 XMM27, XMM27b,
1163 XMM28, XMM28b,
1164 XMM29, XMM29b,
1165 XMM30, XMM30b,
1166 XMM31, XMM31b);
1167
1168 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1169 reg_class_dynamic double_reg_vl(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1170
1171 // Class for pre evex 32bit vector registers
1172 reg_class vectors_reg_legacy(XMM0,
1173 XMM1,
1174 XMM2,
1175 XMM3,
1176 XMM4,
1177 XMM5,
1178 XMM6,
1179 XMM7,
1180 XMM8,
1181 XMM9,
1182 XMM10,
1183 XMM11,
1184 XMM12,
1185 XMM13,
1186 XMM14,
1187 XMM15);
1188
1189 // Class for evex 32bit vector registers
1190 reg_class vectors_reg_evex(XMM0,
1191 XMM1,
1192 XMM2,
1193 XMM3,
1194 XMM4,
1195 XMM5,
1196 XMM6,
1197 XMM7,
1198 XMM8,
1199 XMM9,
1200 XMM10,
1201 XMM11,
1202 XMM12,
1203 XMM13,
1204 XMM14,
1205 XMM15,
1206 XMM16,
1207 XMM17,
1208 XMM18,
1209 XMM19,
1210 XMM20,
1211 XMM21,
1212 XMM22,
1213 XMM23,
1214 XMM24,
1215 XMM25,
1216 XMM26,
1217 XMM27,
1218 XMM28,
1219 XMM29,
1220 XMM30,
1221 XMM31);
1222
1223 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1224 reg_class_dynamic vectors_reg_vlbwdq(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1225
1226 // Class for all 64bit vector registers
1227 reg_class vectord_reg_legacy(XMM0, XMM0b,
1228 XMM1, XMM1b,
1229 XMM2, XMM2b,
1230 XMM3, XMM3b,
1231 XMM4, XMM4b,
1232 XMM5, XMM5b,
1233 XMM6, XMM6b,
1234 XMM7, XMM7b,
1235 XMM8, XMM8b,
1236 XMM9, XMM9b,
1237 XMM10, XMM10b,
1238 XMM11, XMM11b,
1239 XMM12, XMM12b,
1240 XMM13, XMM13b,
1241 XMM14, XMM14b,
1242 XMM15, XMM15b);
1243
1244 // Class for all 64bit vector registers
1245 reg_class vectord_reg_evex(XMM0, XMM0b,
1246 XMM1, XMM1b,
1247 XMM2, XMM2b,
1248 XMM3, XMM3b,
1249 XMM4, XMM4b,
1250 XMM5, XMM5b,
1251 XMM6, XMM6b,
1252 XMM7, XMM7b,
1253 XMM8, XMM8b,
1254 XMM9, XMM9b,
1255 XMM10, XMM10b,
1256 XMM11, XMM11b,
1257 XMM12, XMM12b,
1258 XMM13, XMM13b,
1259 XMM14, XMM14b,
1260 XMM15, XMM15b,
1261 XMM16, XMM16b,
1262 XMM17, XMM17b,
1263 XMM18, XMM18b,
1264 XMM19, XMM19b,
1265 XMM20, XMM20b,
1266 XMM21, XMM21b,
1267 XMM22, XMM22b,
1268 XMM23, XMM23b,
1269 XMM24, XMM24b,
1270 XMM25, XMM25b,
1271 XMM26, XMM26b,
1272 XMM27, XMM27b,
1273 XMM28, XMM28b,
1274 XMM29, XMM29b,
1275 XMM30, XMM30b,
1276 XMM31, XMM31b);
1277
1278 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1279 reg_class_dynamic vectord_reg_vlbwdq(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1280
1281 // Class for all 128bit vector registers
1282 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1283 XMM1, XMM1b, XMM1c, XMM1d,
1284 XMM2, XMM2b, XMM2c, XMM2d,
1285 XMM3, XMM3b, XMM3c, XMM3d,
1286 XMM4, XMM4b, XMM4c, XMM4d,
1287 XMM5, XMM5b, XMM5c, XMM5d,
1288 XMM6, XMM6b, XMM6c, XMM6d,
1289 XMM7, XMM7b, XMM7c, XMM7d,
1290 XMM8, XMM8b, XMM8c, XMM8d,
1291 XMM9, XMM9b, XMM9c, XMM9d,
1292 XMM10, XMM10b, XMM10c, XMM10d,
1293 XMM11, XMM11b, XMM11c, XMM11d,
1294 XMM12, XMM12b, XMM12c, XMM12d,
1295 XMM13, XMM13b, XMM13c, XMM13d,
1296 XMM14, XMM14b, XMM14c, XMM14d,
1297 XMM15, XMM15b, XMM15c, XMM15d);
1298
1299 // Class for all 128bit vector registers
1300 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1301 XMM1, XMM1b, XMM1c, XMM1d,
1302 XMM2, XMM2b, XMM2c, XMM2d,
1303 XMM3, XMM3b, XMM3c, XMM3d,
1304 XMM4, XMM4b, XMM4c, XMM4d,
1305 XMM5, XMM5b, XMM5c, XMM5d,
1306 XMM6, XMM6b, XMM6c, XMM6d,
1307 XMM7, XMM7b, XMM7c, XMM7d,
1308 XMM8, XMM8b, XMM8c, XMM8d,
1309 XMM9, XMM9b, XMM9c, XMM9d,
1310 XMM10, XMM10b, XMM10c, XMM10d,
1311 XMM11, XMM11b, XMM11c, XMM11d,
1312 XMM12, XMM12b, XMM12c, XMM12d,
1313 XMM13, XMM13b, XMM13c, XMM13d,
1314 XMM14, XMM14b, XMM14c, XMM14d,
1315 XMM15, XMM15b, XMM15c, XMM15d,
1316 XMM16, XMM16b, XMM16c, XMM16d,
1317 XMM17, XMM17b, XMM17c, XMM17d,
1318 XMM18, XMM18b, XMM18c, XMM18d,
1319 XMM19, XMM19b, XMM19c, XMM19d,
1320 XMM20, XMM20b, XMM20c, XMM20d,
1321 XMM21, XMM21b, XMM21c, XMM21d,
1322 XMM22, XMM22b, XMM22c, XMM22d,
1323 XMM23, XMM23b, XMM23c, XMM23d,
1324 XMM24, XMM24b, XMM24c, XMM24d,
1325 XMM25, XMM25b, XMM25c, XMM25d,
1326 XMM26, XMM26b, XMM26c, XMM26d,
1327 XMM27, XMM27b, XMM27c, XMM27d,
1328 XMM28, XMM28b, XMM28c, XMM28d,
1329 XMM29, XMM29b, XMM29c, XMM29d,
1330 XMM30, XMM30b, XMM30c, XMM30d,
1331 XMM31, XMM31b, XMM31c, XMM31d);
1332
1333 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1334 reg_class_dynamic vectorx_reg_vlbwdq(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1335
1336 // Class for all 256bit vector registers
1337 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1338 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1339 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1340 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1341 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1342 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1343 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1344 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1345 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1346 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1347 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1348 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1349 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1350 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1351 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1352 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1353
1354 // Class for all 256bit vector registers
1355 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1356 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1357 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1358 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1359 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1360 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1361 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1362 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1363 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1364 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1365 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1366 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1367 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1368 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1369 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1370 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1371 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1372 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1373 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1374 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1375 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1376 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1377 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1378 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1379 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1380 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1381 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1382 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1383 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1384 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1385 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1386 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1387
1388 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1389 reg_class_dynamic vectory_reg_vlbwdq(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1390
1391 // Class for all 512bit vector registers
1392 reg_class vectorz_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1393 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1394 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1395 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1396 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1397 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1398 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1399 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1400 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1401 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1402 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1403 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1404 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1405 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1406 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1407 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1408 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1409 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1410 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1411 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1412 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1413 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1414 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1415 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1416 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1417 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1418 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1419 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1420 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1421 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1422 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1423 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1424
1425 // Class for restricted 512bit vector registers
1426 reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1427 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1428 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1429 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1430 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1431 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1432 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1433 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1434 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1435 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1436 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1437 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1438 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1439 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1440 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1441 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1442
1443 reg_class_dynamic vectorz_reg (vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
1444 reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1445
1446 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1447
1448 %}
1449
1450
1451 //----------SOURCE BLOCK-------------------------------------------------------
1452 // This is a block of C++ code which provides values, functions, and
1453 // definitions necessary in the rest of the architecture description
1454
1455 source_hpp %{
1456
1457 #include "peephole_x86_64.hpp"
1458
1459 bool castLL_is_imm32(const Node* n);
1460
1461 %}
1462
1463 source %{
1464
1465 bool castLL_is_imm32(const Node* n) {
1466 assert(n->is_CastLL(), "must be a CastLL");
1467 const TypeLong* t = n->bottom_type()->is_long();
1468 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
1469 }
1470
1471 %}
1472
1473 // Register masks
1474 source_hpp %{
1475
1476 extern RegMask _ANY_REG_mask;
1477 extern RegMask _PTR_REG_mask;
1478 extern RegMask _PTR_REG_NO_RBP_mask;
1479 extern RegMask _PTR_NO_RAX_REG_mask;
1480 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
1481 extern RegMask _LONG_REG_mask;
1482 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
1483 extern RegMask _LONG_NO_RCX_REG_mask;
1484 extern RegMask _LONG_NO_RBP_R13_REG_mask;
1485 extern RegMask _INT_REG_mask;
1486 extern RegMask _INT_NO_RAX_RDX_REG_mask;
1487 extern RegMask _INT_NO_RCX_REG_mask;
1488 extern RegMask _INT_NO_RBP_R13_REG_mask;
1489 extern RegMask _FLOAT_REG_mask;
1490
1491 extern RegMask _STACK_OR_PTR_REG_mask;
1492 extern RegMask _STACK_OR_LONG_REG_mask;
1493 extern RegMask _STACK_OR_INT_REG_mask;
1494
1495 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
1496 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
1497 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
1498
1499 %}
1500
1501 source %{
1502 #define RELOC_IMM64 Assembler::imm_operand
1503 #define RELOC_DISP32 Assembler::disp32_operand
1504
1505 #define __ masm->
1506
1507 RegMask _ANY_REG_mask;
1508 RegMask _PTR_REG_mask;
1509 RegMask _PTR_REG_NO_RBP_mask;
1510 RegMask _PTR_NO_RAX_REG_mask;
1511 RegMask _PTR_NO_RAX_RBX_REG_mask;
1512 RegMask _LONG_REG_mask;
1513 RegMask _LONG_NO_RAX_RDX_REG_mask;
1514 RegMask _LONG_NO_RCX_REG_mask;
1515 RegMask _LONG_NO_RBP_R13_REG_mask;
1516 RegMask _INT_REG_mask;
1517 RegMask _INT_NO_RAX_RDX_REG_mask;
1518 RegMask _INT_NO_RCX_REG_mask;
1519 RegMask _INT_NO_RBP_R13_REG_mask;
1520 RegMask _FLOAT_REG_mask;
1521 RegMask _STACK_OR_PTR_REG_mask;
1522 RegMask _STACK_OR_LONG_REG_mask;
1523 RegMask _STACK_OR_INT_REG_mask;
1524
1525 static bool need_r12_heapbase() {
1526 return UseCompressedOops;
1527 }
1528
1529 void reg_mask_init() {
1530 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
1531
1532 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
1533 // We derive a number of subsets from it.
1534 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1535
1536 if (PreserveFramePointer) {
1537 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1538 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1539 }
1540 if (need_r12_heapbase()) {
1541 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1542 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
1543 }
1544
1545 _PTR_REG_mask.assignFrom(_ANY_REG_mask);
1546 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
1547 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
1548 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
1549 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
1550 if (!UseAPX) {
1551 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1552 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1553 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
1554 }
1555 }
1556
1557 _STACK_OR_PTR_REG_mask.assignFrom(_PTR_REG_mask);
1558 _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1559
1560 _PTR_REG_NO_RBP_mask.assignFrom(_PTR_REG_mask);
1561 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1562 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1563
1564 _PTR_NO_RAX_REG_mask.assignFrom(_PTR_REG_mask);
1565 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1566 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1567
1568 _PTR_NO_RAX_RBX_REG_mask.assignFrom(_PTR_NO_RAX_REG_mask);
1569 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
1570 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
1571
1572
1573 _LONG_REG_mask.assignFrom(_PTR_REG_mask);
1574 _STACK_OR_LONG_REG_mask.assignFrom(_LONG_REG_mask);
1575 _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1576
1577 _LONG_NO_RAX_RDX_REG_mask.assignFrom(_LONG_REG_mask);
1578 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1579 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1580 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1581 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
1582
1583 _LONG_NO_RCX_REG_mask.assignFrom(_LONG_REG_mask);
1584 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1585 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
1586
1587 _LONG_NO_RBP_R13_REG_mask.assignFrom(_LONG_REG_mask);
1588 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1589 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1590 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1591 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
1592
1593 _INT_REG_mask.assignFrom(_ALL_INT_REG_mask);
1594 if (!UseAPX) {
1595 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1596 _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1597 }
1598 }
1599
1600 if (PreserveFramePointer) {
1601 _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1602 }
1603 if (need_r12_heapbase()) {
1604 _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1605 }
1606
1607 _STACK_OR_INT_REG_mask.assignFrom(_INT_REG_mask);
1608 _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1609
1610 _INT_NO_RAX_RDX_REG_mask.assignFrom(_INT_REG_mask);
1611 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1612 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1613
1614 _INT_NO_RCX_REG_mask.assignFrom(_INT_REG_mask);
1615 _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1616
1617 _INT_NO_RBP_R13_REG_mask.assignFrom(_INT_REG_mask);
1618 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1619 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1620
1621 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
1622 // from the float_reg_legacy/float_reg_evex register class.
1623 _FLOAT_REG_mask.assignFrom(VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask);
1624 }
1625
1626 static bool generate_vzeroupper(Compile* C) {
1627 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
1628 }
1629
1630 static int clear_avx_size() {
1631 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset()
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() {
1652 if (_entry_point == nullptr) {
1653 // CallLeafNoFPInDirect
1654 return 3; // callq (register)
1655 }
1656 int offset = 13; // movq r10,#addr; callq (r10)
1657 if (this->ideal_Opcode() != Op_CallLeafVector) {
1658 offset += clear_avx_size();
1659 }
1660 return offset;
1661 }
1662 //
1663 // Compute padding required for nodes which need alignment
1664 //
1665
1666 // The address of the call instruction needs to be 4-byte aligned to
1667 // ensure that it does not span a cache line so that it can be patched.
1668 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1669 {
1670 current_offset += clear_avx_size(); // skip vzeroupper
1671 current_offset += 1; // skip call opcode byte
1672 return align_up(current_offset, alignment_required()) - current_offset;
1673 }
1674
1675 // The address of the call instruction needs to be 4-byte aligned to
1676 // ensure that it does not span a cache line so that it can be patched.
1677 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
1678 {
1679 current_offset += clear_avx_size(); // skip vzeroupper
1680 current_offset += 11; // skip movq instruction + call opcode byte
1681 return align_up(current_offset, alignment_required()) - current_offset;
1682 }
1683
1684 // This could be in MacroAssembler but it's fairly C2 specific
1685 static void emit_cmpfp_fixup(MacroAssembler* masm) {
1686 Label exit;
1687 __ jccb(Assembler::noParity, exit);
1688 __ pushf();
1689 //
1690 // comiss/ucomiss instructions set ZF,PF,CF flags and
1691 // zero OF,AF,SF for NaN values.
1692 // Fixup flags by zeroing ZF,PF so that compare of NaN
1693 // values returns 'less than' result (CF is set).
1694 // Leave the rest of flags unchanged.
1695 //
1696 // 7 6 5 4 3 2 1 0
1697 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
1698 // 0 0 1 0 1 0 1 1 (0x2B)
1699 //
1700 __ andq(Address(rsp, 0), 0xffffff2b);
1701 __ popf();
1702 __ bind(exit);
1703 }
1704
1705 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
1706 // If any floating point comparison instruction is used, unordered case always triggers jump
1707 // for below condition, CF=1 is true when at least one input is NaN
1708 Label done;
1709 __ movl(dst, -1);
1710 __ jcc(Assembler::below, done);
1711 __ setcc(Assembler::notEqual, dst);
1712 __ bind(done);
1713 }
1714
1715 enum FP_PREC {
1716 fp_prec_hlf,
1717 fp_prec_flt,
1718 fp_prec_dbl
1719 };
1720
1721 static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
1722 XMMRegister p, XMMRegister q) {
1723 if (pt == fp_prec_hlf) {
1724 __ evucomish(p, q);
1725 } else if (pt == fp_prec_flt) {
1726 __ ucomiss(p, q);
1727 } else {
1728 __ ucomisd(p, q);
1729 }
1730 }
1731
1732 static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
1733 XMMRegister dst, XMMRegister src, Register scratch) {
1734 if (pt == fp_prec_hlf) {
1735 __ movhlf(dst, src, scratch);
1736 } else if (pt == fp_prec_flt) {
1737 __ movflt(dst, src);
1738 } else {
1739 __ movdbl(dst, src);
1740 }
1741 }
1742
1743 // Math.min() # Math.max()
1744 // -----------------------------
1745 // (v)ucomis[h/s/d] #
1746 // ja -> b # a
1747 // jp -> NaN # NaN
1748 // jb -> a # b
1749 // je -> a | b # a & b
1750 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
1751 XMMRegister a, XMMRegister b, Register rt,
1752 bool min, enum FP_PREC pt) {
1753 Label nan, zero, below, above, done;
1754
1755 emit_fp_ucom(masm, pt, a, b);
1756
1757 if (dst->encoding() != (min ? b : a)->encoding()) {
1758 __ jccb(Assembler::above, above); // CF=0 & ZF=0
1759 } else {
1760 __ jccb(Assembler::above, done);
1761 }
1762 __ jccb(Assembler::parity, nan); // PF=1
1763 __ jccb(Assembler::below, below); // CF=1
1764
1765 // equal
1766 // Using bitwise operations is a low cost way to compute the correct result
1767 // for zero and non-zero inputs in this scenario except for NaN, which is
1768 // handled separately. The mantissa and exponent are valid with either
1769 // bitwise operation. For zero inputs, the sign bit is chosen according to
1770 // whether a minimum or maximum value is required.
1771 if (min) {
1772 // Negative sign preserved when available (e.g., min(+0, -0) -> -0)
1773 __ vpor(dst, a, b, Assembler::AVX_128bit);
1774 } else {
1775 // Positive sign preserved when available (e.g., max(+0, -0) -> +0)
1776 __ vpand(dst, a, b, Assembler::AVX_128bit);
1777 }
1778 __ jmp(done);
1779
1780 __ bind(above);
1781 movfp(masm, pt, dst, min ? b : a, rt);
1782 __ jmp(done);
1783
1784 __ bind(nan);
1785 if (pt == fp_prec_hlf) {
1786 __ movl(rt, 0x00007e00); // Float16.NaN
1787 __ evmovw(dst, rt);
1788 } else if (pt == fp_prec_flt) {
1789 __ movl(rt, 0x7fc00000); // Float.NaN
1790 __ movdl(dst, rt);
1791 } else {
1792 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
1793 __ movdq(dst, rt);
1794 }
1795 __ jmp(done);
1796
1797 __ bind(below);
1798 movfp(masm, pt, dst, min ? a : b, rt);
1799
1800 __ bind(done);
1801 }
1802
1803 //=============================================================================
1804 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1805
1806 int ConstantTable::calculate_table_base_offset() const {
1807 return 0; // absolute addressing, no offset
1808 }
1809
1810 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1811 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1812 ShouldNotReachHere();
1813 }
1814
1815 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1816 // Empty encoding
1817 }
1818
1819 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1820 return 0;
1821 }
1822
1823 #ifndef PRODUCT
1824 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1825 st->print("# MachConstantBaseNode (empty encoding)");
1826 }
1827 #endif
1828
1829
1830 //=============================================================================
1831 #ifndef PRODUCT
1832 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1833 Compile* C = ra_->C;
1834
1835 int framesize = C->output()->frame_size_in_bytes();
1836 int bangsize = C->output()->bang_size_in_bytes();
1837 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1838 // Remove wordSize for return addr which is already pushed.
1839 framesize -= wordSize;
1840
1841 if (C->output()->need_stack_bang(bangsize)) {
1842 framesize -= wordSize;
1843 st->print("# stack bang (%d bytes)", bangsize);
1844 st->print("\n\t");
1845 st->print("pushq rbp\t# Save rbp");
1846 if (PreserveFramePointer) {
1847 st->print("\n\t");
1848 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1849 }
1850 if (framesize) {
1851 st->print("\n\t");
1852 st->print("subq rsp, #%d\t# Create frame",framesize);
1853 }
1854 } else {
1855 st->print("subq rsp, #%d\t# Create frame",framesize);
1856 st->print("\n\t");
1857 framesize -= wordSize;
1858 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
1859 if (PreserveFramePointer) {
1860 st->print("\n\t");
1861 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1862 if (framesize > 0) {
1863 st->print("\n\t");
1864 st->print("addq rbp, #%d", framesize);
1865 }
1866 }
1867 }
1868
1869 if (VerifyStackAtCalls) {
1870 st->print("\n\t");
1871 framesize -= wordSize;
1872 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1873 #ifdef ASSERT
1874 st->print("\n\t");
1875 st->print("# stack alignment check");
1876 #endif
1877 }
1878 if (C->stub_function() != nullptr) {
1879 st->print("\n\t");
1880 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1881 st->print("\n\t");
1882 st->print("je fast_entry\t");
1883 st->print("\n\t");
1884 st->print("call #nmethod_entry_barrier_stub\t");
1885 st->print("\n\tfast_entry:");
1886 }
1887 st->cr();
1888 }
1889 #endif
1890
1891 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1892 Compile* C = ra_->C;
1893
1894 __ verified_entry(C);
1895
1896 if (ra_->C->stub_function() == nullptr) {
1897 __ entry_barrier();
1898 }
1899
1900 if (!Compile::current()->output()->in_scratch_emit_size()) {
1901 __ bind(*_verified_entry);
1902 }
1903
1904 C->output()->set_frame_complete(__ offset());
1905
1906 if (C->has_mach_constant_base_node()) {
1907 // NOTE: We set the table base offset here because users might be
1908 // emitted before MachConstantBaseNode.
1909 ConstantTable& constant_table = C->output()->constant_table();
1910 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1911 }
1912 }
1913
1914
1915 int MachPrologNode::reloc() const
1916 {
1917 return 0; // a large enough number
1918 }
1919
1920 //=============================================================================
1921 #ifndef PRODUCT
1922 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1923 {
1924 Compile* C = ra_->C;
1925 if (generate_vzeroupper(C)) {
1926 st->print("vzeroupper");
1927 st->cr(); st->print("\t");
1928 }
1929
1930 int framesize = C->output()->frame_size_in_bytes();
1931 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1932 // Remove word for return adr already pushed
1933 // and RBP
1934 framesize -= 2*wordSize;
1935
1936 if (framesize) {
1937 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
1938 st->print("\t");
1939 }
1940
1941 st->print_cr("popq rbp");
1942 if (do_polling() && C->is_method_compilation()) {
1943 st->print("\t");
1944 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1945 "ja #safepoint_stub\t"
1946 "# Safepoint: poll for GC");
1947 }
1948 }
1949 #endif
1950
1951 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1952 {
1953 Compile* C = ra_->C;
1954
1955 if (generate_vzeroupper(C)) {
1956 // Clear upper bits of YMM registers when current compiled code uses
1957 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1958 __ vzeroupper();
1959 }
1960
1961 // Subtract two words to account for return address and rbp
1962 int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
1963 __ remove_frame(initial_framesize, C->needs_stack_repair());
1964
1965 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1966 __ reserved_stack_check();
1967 }
1968
1969 if (do_polling() && C->is_method_compilation()) {
1970 Label dummy_label;
1971 Label* code_stub = &dummy_label;
1972 if (!C->output()->in_scratch_emit_size()) {
1973 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1974 C->output()->add_stub(stub);
1975 code_stub = &stub->entry();
1976 }
1977 __ relocate(relocInfo::poll_return_type);
1978 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1979 }
1980 }
1981
1982 int MachEpilogNode::reloc() const
1983 {
1984 return 2; // a large enough number
1985 }
1986
1987 const Pipeline* MachEpilogNode::pipeline() const
1988 {
1989 return MachNode::pipeline_class();
1990 }
1991
1992 //=============================================================================
1993
1994 enum RC {
1995 rc_bad,
1996 rc_int,
1997 rc_kreg,
1998 rc_float,
1999 rc_stack
2000 };
2001
2002 static enum RC rc_class(OptoReg::Name reg)
2003 {
2004 if( !OptoReg::is_valid(reg) ) return rc_bad;
2005
2006 if (OptoReg::is_stack(reg)) return rc_stack;
2007
2008 VMReg r = OptoReg::as_VMReg(reg);
2009
2010 if (r->is_Register()) return rc_int;
2011
2012 if (r->is_KRegister()) return rc_kreg;
2013
2014 assert(r->is_XMMRegister(), "must be");
2015 return rc_float;
2016 }
2017
2018 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
2019 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
2020 int src_hi, int dst_hi, uint ireg, outputStream* st);
2021
2022 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
2023 int stack_offset, int reg, uint ireg, outputStream* st);
2024
2025 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
2026 int dst_offset, uint ireg, outputStream* st) {
2027 if (masm) {
2028 switch (ireg) {
2029 case Op_VecS:
2030 __ movq(Address(rsp, -8), rax);
2031 __ movl(rax, Address(rsp, src_offset));
2032 __ movl(Address(rsp, dst_offset), rax);
2033 __ movq(rax, Address(rsp, -8));
2034 break;
2035 case Op_VecD:
2036 __ pushq(Address(rsp, src_offset));
2037 __ popq (Address(rsp, dst_offset));
2038 break;
2039 case Op_VecX:
2040 __ pushq(Address(rsp, src_offset));
2041 __ popq (Address(rsp, dst_offset));
2042 __ pushq(Address(rsp, src_offset+8));
2043 __ popq (Address(rsp, dst_offset+8));
2044 break;
2045 case Op_VecY:
2046 __ vmovdqu(Address(rsp, -32), xmm0);
2047 __ vmovdqu(xmm0, Address(rsp, src_offset));
2048 __ vmovdqu(Address(rsp, dst_offset), xmm0);
2049 __ vmovdqu(xmm0, Address(rsp, -32));
2050 break;
2051 case Op_VecZ:
2052 __ evmovdquq(Address(rsp, -64), xmm0, 2);
2053 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
2054 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
2055 __ evmovdquq(xmm0, Address(rsp, -64), 2);
2056 break;
2057 default:
2058 ShouldNotReachHere();
2059 }
2060 #ifndef PRODUCT
2061 } else {
2062 switch (ireg) {
2063 case Op_VecS:
2064 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2065 "movl rax, [rsp + #%d]\n\t"
2066 "movl [rsp + #%d], rax\n\t"
2067 "movq rax, [rsp - #8]",
2068 src_offset, dst_offset);
2069 break;
2070 case Op_VecD:
2071 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2072 "popq [rsp + #%d]",
2073 src_offset, dst_offset);
2074 break;
2075 case Op_VecX:
2076 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
2077 "popq [rsp + #%d]\n\t"
2078 "pushq [rsp + #%d]\n\t"
2079 "popq [rsp + #%d]",
2080 src_offset, dst_offset, src_offset+8, dst_offset+8);
2081 break;
2082 case Op_VecY:
2083 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
2084 "vmovdqu xmm0, [rsp + #%d]\n\t"
2085 "vmovdqu [rsp + #%d], xmm0\n\t"
2086 "vmovdqu xmm0, [rsp - #32]",
2087 src_offset, dst_offset);
2088 break;
2089 case Op_VecZ:
2090 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
2091 "vmovdqu xmm0, [rsp + #%d]\n\t"
2092 "vmovdqu [rsp + #%d], xmm0\n\t"
2093 "vmovdqu xmm0, [rsp - #64]",
2094 src_offset, dst_offset);
2095 break;
2096 default:
2097 ShouldNotReachHere();
2098 }
2099 #endif
2100 }
2101 }
2102
2103 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
2104 PhaseRegAlloc* ra_,
2105 bool do_size,
2106 outputStream* st) const {
2107 assert(masm != nullptr || st != nullptr, "sanity");
2108 // Get registers to move
2109 OptoReg::Name src_second = ra_->get_reg_second(in(1));
2110 OptoReg::Name src_first = ra_->get_reg_first(in(1));
2111 OptoReg::Name dst_second = ra_->get_reg_second(this);
2112 OptoReg::Name dst_first = ra_->get_reg_first(this);
2113
2114 enum RC src_second_rc = rc_class(src_second);
2115 enum RC src_first_rc = rc_class(src_first);
2116 enum RC dst_second_rc = rc_class(dst_second);
2117 enum RC dst_first_rc = rc_class(dst_first);
2118
2119 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
2120 "must move at least 1 register" );
2121
2122 if (src_first == dst_first && src_second == dst_second) {
2123 // Self copy, no move
2124 return 0;
2125 }
2126 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
2127 uint ireg = ideal_reg();
2128 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
2129 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
2130 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
2131 // mem -> mem
2132 int src_offset = ra_->reg2offset(src_first);
2133 int dst_offset = ra_->reg2offset(dst_first);
2134 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
2135 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
2136 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
2137 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
2138 int stack_offset = ra_->reg2offset(dst_first);
2139 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
2140 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
2141 int stack_offset = ra_->reg2offset(src_first);
2142 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
2143 } else {
2144 ShouldNotReachHere();
2145 }
2146 return 0;
2147 }
2148 if (src_first_rc == rc_stack) {
2149 // mem ->
2150 if (dst_first_rc == rc_stack) {
2151 // mem -> mem
2152 assert(src_second != dst_first, "overlap");
2153 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2154 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2155 // 64-bit
2156 int src_offset = ra_->reg2offset(src_first);
2157 int dst_offset = ra_->reg2offset(dst_first);
2158 if (masm) {
2159 __ pushq(Address(rsp, src_offset));
2160 __ popq (Address(rsp, dst_offset));
2161 #ifndef PRODUCT
2162 } else {
2163 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2164 "popq [rsp + #%d]",
2165 src_offset, dst_offset);
2166 #endif
2167 }
2168 } else {
2169 // 32-bit
2170 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2171 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2172 // No pushl/popl, so:
2173 int src_offset = ra_->reg2offset(src_first);
2174 int dst_offset = ra_->reg2offset(dst_first);
2175 if (masm) {
2176 __ movq(Address(rsp, -8), rax);
2177 __ movl(rax, Address(rsp, src_offset));
2178 __ movl(Address(rsp, dst_offset), rax);
2179 __ movq(rax, Address(rsp, -8));
2180 #ifndef PRODUCT
2181 } else {
2182 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2183 "movl rax, [rsp + #%d]\n\t"
2184 "movl [rsp + #%d], rax\n\t"
2185 "movq rax, [rsp - #8]",
2186 src_offset, dst_offset);
2187 #endif
2188 }
2189 }
2190 return 0;
2191 } else if (dst_first_rc == rc_int) {
2192 // mem -> gpr
2193 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2194 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2195 // 64-bit
2196 int offset = ra_->reg2offset(src_first);
2197 if (masm) {
2198 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2199 #ifndef PRODUCT
2200 } else {
2201 st->print("movq %s, [rsp + #%d]\t# spill",
2202 Matcher::regName[dst_first],
2203 offset);
2204 #endif
2205 }
2206 } else {
2207 // 32-bit
2208 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2209 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2210 int offset = ra_->reg2offset(src_first);
2211 if (masm) {
2212 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2213 #ifndef PRODUCT
2214 } else {
2215 st->print("movl %s, [rsp + #%d]\t# spill",
2216 Matcher::regName[dst_first],
2217 offset);
2218 #endif
2219 }
2220 }
2221 return 0;
2222 } else if (dst_first_rc == rc_float) {
2223 // mem-> xmm
2224 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2225 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2226 // 64-bit
2227 int offset = ra_->reg2offset(src_first);
2228 if (masm) {
2229 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2230 #ifndef PRODUCT
2231 } else {
2232 st->print("%s %s, [rsp + #%d]\t# spill",
2233 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
2234 Matcher::regName[dst_first],
2235 offset);
2236 #endif
2237 }
2238 } else {
2239 // 32-bit
2240 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2241 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2242 int offset = ra_->reg2offset(src_first);
2243 if (masm) {
2244 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2245 #ifndef PRODUCT
2246 } else {
2247 st->print("movss %s, [rsp + #%d]\t# spill",
2248 Matcher::regName[dst_first],
2249 offset);
2250 #endif
2251 }
2252 }
2253 return 0;
2254 } else if (dst_first_rc == rc_kreg) {
2255 // mem -> kreg
2256 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2257 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2258 // 64-bit
2259 int offset = ra_->reg2offset(src_first);
2260 if (masm) {
2261 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2262 #ifndef PRODUCT
2263 } else {
2264 st->print("kmovq %s, [rsp + #%d]\t# spill",
2265 Matcher::regName[dst_first],
2266 offset);
2267 #endif
2268 }
2269 }
2270 return 0;
2271 }
2272 } else if (src_first_rc == rc_int) {
2273 // gpr ->
2274 if (dst_first_rc == rc_stack) {
2275 // gpr -> mem
2276 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2277 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2278 // 64-bit
2279 int offset = ra_->reg2offset(dst_first);
2280 if (masm) {
2281 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2282 #ifndef PRODUCT
2283 } else {
2284 st->print("movq [rsp + #%d], %s\t# spill",
2285 offset,
2286 Matcher::regName[src_first]);
2287 #endif
2288 }
2289 } else {
2290 // 32-bit
2291 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2292 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2293 int offset = ra_->reg2offset(dst_first);
2294 if (masm) {
2295 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2296 #ifndef PRODUCT
2297 } else {
2298 st->print("movl [rsp + #%d], %s\t# spill",
2299 offset,
2300 Matcher::regName[src_first]);
2301 #endif
2302 }
2303 }
2304 return 0;
2305 } else if (dst_first_rc == rc_int) {
2306 // gpr -> gpr
2307 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2308 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2309 // 64-bit
2310 if (masm) {
2311 __ movq(as_Register(Matcher::_regEncode[dst_first]),
2312 as_Register(Matcher::_regEncode[src_first]));
2313 #ifndef PRODUCT
2314 } else {
2315 st->print("movq %s, %s\t# spill",
2316 Matcher::regName[dst_first],
2317 Matcher::regName[src_first]);
2318 #endif
2319 }
2320 return 0;
2321 } else {
2322 // 32-bit
2323 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2324 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2325 if (masm) {
2326 __ movl(as_Register(Matcher::_regEncode[dst_first]),
2327 as_Register(Matcher::_regEncode[src_first]));
2328 #ifndef PRODUCT
2329 } else {
2330 st->print("movl %s, %s\t# spill",
2331 Matcher::regName[dst_first],
2332 Matcher::regName[src_first]);
2333 #endif
2334 }
2335 return 0;
2336 }
2337 } else if (dst_first_rc == rc_float) {
2338 // gpr -> xmm
2339 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2340 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2341 // 64-bit
2342 if (masm) {
2343 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2344 #ifndef PRODUCT
2345 } else {
2346 st->print("movdq %s, %s\t# spill",
2347 Matcher::regName[dst_first],
2348 Matcher::regName[src_first]);
2349 #endif
2350 }
2351 } else {
2352 // 32-bit
2353 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2354 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2355 if (masm) {
2356 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2357 #ifndef PRODUCT
2358 } else {
2359 st->print("movdl %s, %s\t# spill",
2360 Matcher::regName[dst_first],
2361 Matcher::regName[src_first]);
2362 #endif
2363 }
2364 }
2365 return 0;
2366 } else if (dst_first_rc == rc_kreg) {
2367 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2368 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2369 // 64-bit
2370 if (masm) {
2371 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2372 #ifndef PRODUCT
2373 } else {
2374 st->print("kmovq %s, %s\t# spill",
2375 Matcher::regName[dst_first],
2376 Matcher::regName[src_first]);
2377 #endif
2378 }
2379 }
2380 Unimplemented();
2381 return 0;
2382 }
2383 } else if (src_first_rc == rc_float) {
2384 // xmm ->
2385 if (dst_first_rc == rc_stack) {
2386 // xmm -> mem
2387 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2388 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2389 // 64-bit
2390 int offset = ra_->reg2offset(dst_first);
2391 if (masm) {
2392 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2393 #ifndef PRODUCT
2394 } else {
2395 st->print("movsd [rsp + #%d], %s\t# spill",
2396 offset,
2397 Matcher::regName[src_first]);
2398 #endif
2399 }
2400 } else {
2401 // 32-bit
2402 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2403 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2404 int offset = ra_->reg2offset(dst_first);
2405 if (masm) {
2406 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2407 #ifndef PRODUCT
2408 } else {
2409 st->print("movss [rsp + #%d], %s\t# spill",
2410 offset,
2411 Matcher::regName[src_first]);
2412 #endif
2413 }
2414 }
2415 return 0;
2416 } else if (dst_first_rc == rc_int) {
2417 // xmm -> gpr
2418 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2419 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2420 // 64-bit
2421 if (masm) {
2422 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2423 #ifndef PRODUCT
2424 } else {
2425 st->print("movdq %s, %s\t# spill",
2426 Matcher::regName[dst_first],
2427 Matcher::regName[src_first]);
2428 #endif
2429 }
2430 } else {
2431 // 32-bit
2432 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2433 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2434 if (masm) {
2435 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2436 #ifndef PRODUCT
2437 } else {
2438 st->print("movdl %s, %s\t# spill",
2439 Matcher::regName[dst_first],
2440 Matcher::regName[src_first]);
2441 #endif
2442 }
2443 }
2444 return 0;
2445 } else if (dst_first_rc == rc_float) {
2446 // xmm -> xmm
2447 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2448 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2449 // 64-bit
2450 if (masm) {
2451 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2452 #ifndef PRODUCT
2453 } else {
2454 st->print("%s %s, %s\t# spill",
2455 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
2456 Matcher::regName[dst_first],
2457 Matcher::regName[src_first]);
2458 #endif
2459 }
2460 } else {
2461 // 32-bit
2462 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2463 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2464 if (masm) {
2465 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2466 #ifndef PRODUCT
2467 } else {
2468 st->print("%s %s, %s\t# spill",
2469 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
2470 Matcher::regName[dst_first],
2471 Matcher::regName[src_first]);
2472 #endif
2473 }
2474 }
2475 return 0;
2476 } else if (dst_first_rc == rc_kreg) {
2477 assert(false, "Illegal spilling");
2478 return 0;
2479 }
2480 } else if (src_first_rc == rc_kreg) {
2481 if (dst_first_rc == rc_stack) {
2482 // mem -> kreg
2483 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2484 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2485 // 64-bit
2486 int offset = ra_->reg2offset(dst_first);
2487 if (masm) {
2488 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
2489 #ifndef PRODUCT
2490 } else {
2491 st->print("kmovq [rsp + #%d] , %s\t# spill",
2492 offset,
2493 Matcher::regName[src_first]);
2494 #endif
2495 }
2496 }
2497 return 0;
2498 } else if (dst_first_rc == rc_int) {
2499 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2500 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2501 // 64-bit
2502 if (masm) {
2503 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2504 #ifndef PRODUCT
2505 } else {
2506 st->print("kmovq %s, %s\t# spill",
2507 Matcher::regName[dst_first],
2508 Matcher::regName[src_first]);
2509 #endif
2510 }
2511 }
2512 Unimplemented();
2513 return 0;
2514 } else if (dst_first_rc == rc_kreg) {
2515 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2516 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2517 // 64-bit
2518 if (masm) {
2519 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2520 #ifndef PRODUCT
2521 } else {
2522 st->print("kmovq %s, %s\t# spill",
2523 Matcher::regName[dst_first],
2524 Matcher::regName[src_first]);
2525 #endif
2526 }
2527 }
2528 return 0;
2529 } else if (dst_first_rc == rc_float) {
2530 assert(false, "Illegal spill");
2531 return 0;
2532 }
2533 }
2534
2535 assert(0," foo ");
2536 Unimplemented();
2537 return 0;
2538 }
2539
2540 #ifndef PRODUCT
2541 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
2542 implementation(nullptr, ra_, false, st);
2543 }
2544 #endif
2545
2546 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2547 implementation(masm, ra_, false, nullptr);
2548 }
2549
2550 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2551 return MachNode::size(ra_);
2552 }
2553
2554 //=============================================================================
2555 #ifndef PRODUCT
2556 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2557 {
2558 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2559 int reg = ra_->get_reg_first(this);
2560 st->print("leaq %s, [rsp + #%d]\t# box lock",
2561 Matcher::regName[reg], offset);
2562 }
2563 #endif
2564
2565 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2566 {
2567 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2568 int reg = ra_->get_encode(this);
2569
2570 __ lea(as_Register(reg), Address(rsp, offset));
2571 }
2572
2573 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2574 {
2575 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2576 if (ra_->get_encode(this) > 15) {
2577 return (offset < 0x80) ? 6 : 9; // REX2
2578 } else {
2579 return (offset < 0x80) ? 5 : 8; // REX
2580 }
2581 }
2582
2583 //=============================================================================
2584 #ifndef PRODUCT
2585 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2586 {
2587 st->print_cr("MachVEPNode");
2588 }
2589 #endif
2590
2591 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2592 {
2593 CodeBuffer* cbuf = masm->code();
2594 if (!_verified) {
2595 __ ic_check(1);
2596 } else {
2597 if (ra_->C->stub_function() == nullptr) {
2598 // Emit the entry barrier in a temporary frame before unpacking because
2599 // it can deopt, which would require packing the scalarized args again.
2600 __ verified_entry(ra_->C, 0);
2601 __ entry_barrier();
2602 int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
2603 __ remove_frame(initial_framesize, false);
2604 }
2605 // Unpack inline type args passed as oop and then jump to
2606 // the verified entry point (skipping the unverified entry).
2607 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2608 // Emit code for verified entry and save increment for stack repair on return
2609 __ verified_entry(ra_->C, sp_inc);
2610 if (Compile::current()->output()->in_scratch_emit_size()) {
2611 Label dummy_verified_entry;
2612 __ jmp(dummy_verified_entry);
2613 } else {
2614 __ jmp(*_verified_entry);
2615 }
2616 }
2617 if (ra_->C->stub_function() == nullptr) {
2618 // Pad so that the next call to MachVEPNode::emit() starts out with the
2619 // correct alignment. This is needed by entry_barrier() to align the
2620 // compare. But unfortunately we need to align all 4 MachVEPNodes because
2621 // entry point offsets are computed using scratch_emit_size(), so starting
2622 // alignment must match the alignment of the scratch buffer, otherwise the sizes
2623 // will be off.
2624 __ align(4);
2625 }
2626 }
2627
2628 //=============================================================================
2629 #ifndef PRODUCT
2630 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2631 {
2632 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2633 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2634 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2635 }
2636 #endif
2637
2638 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2639 {
2640 __ ic_check(InteriorEntryAlignment);
2641 }
2642
2643
2644 //=============================================================================
2645
2646 bool Matcher::supports_vector_calling_convention(void) {
2647 return EnableVectorSupport;
2648 }
2649
2650 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2651 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2652 }
2653
2654 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2655 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2656 }
2657
2658 #ifdef ASSERT
2659 static bool is_ndd_demotable(const MachNode* mdef) {
2660 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2661 }
2662 #endif
2663
2664 bool Matcher::is_register_biasing_candidate(const MachNode* mdef,
2665 int oper_index) {
2666 if (mdef == nullptr) {
2667 return false;
2668 }
2669
2670 if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 ||
2671 mdef->in(mdef->operand_index(oper_index)) == nullptr) {
2672 assert(oper_index != 1 || !is_ndd_demotable_opr1(mdef), "%s", mdef->Name());
2673 assert(oper_index != 2 || !is_ndd_demotable_opr2(mdef), "%s", mdef->Name());
2674 return false;
2675 }
2676
2677 // Complex memory operand covers multiple incoming edges needed for
2678 // address computation. Biasing def towards any address component will not
2679 // result in NDD demotion by assembler.
2680 if (mdef->operand_num_edges(oper_index) != 1) {
2681 return false;
2682 }
2683
2684 // Demotion candidate must be register mask compatible with definition.
2685 const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index));
2686 if (!oper_mask.overlap(mdef->out_RegMask())) {
2687 assert(!is_ndd_demotable(mdef), "%s", mdef->Name());
2688 return false;
2689 }
2690
2691 switch (oper_index) {
2692 // First operand of MachNode corresponding to Intel APX NDD selection
2693 // pattern can share its assigned register with definition operand if
2694 // their live ranges do not overlap. In such a scenario we can demote
2695 // it to legacy map0/map1 instruction by replacing its 4-byte extended
2696 // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates
2697 // are decorated with a special flag by instruction selector.
2698 case 1:
2699 return is_ndd_demotable_opr1(mdef);
2700
2701 // Definition operand of commutative operation can be biased towards second
2702 // operand.
2703 case 2:
2704 return is_ndd_demotable_opr2(mdef);
2705
2706 // Current scheme only selects up to two biasing candidates
2707 default:
2708 assert(false, "unhandled operand index: %s", mdef->Name());
2709 break;
2710 }
2711
2712 return false;
2713 }
2714
2715 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2716 assert(EnableVectorSupport, "sanity");
2717 int lo = XMM0_num;
2718 int hi = XMM0b_num;
2719 if (ideal_reg == Op_VecX) hi = XMM0d_num;
2720 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
2721 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
2722 return OptoRegPair(hi, lo);
2723 }
2724
2725 // Is this branch offset short enough that a short branch can be used?
2726 //
2727 // NOTE: If the platform does not provide any short branch variants, then
2728 // this method should return false for offset 0.
2729 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2730 // The passed offset is relative to address of the branch.
2731 // On 86 a branch displacement is calculated relative to address
2732 // of a next instruction.
2733 offset -= br_size;
2734
2735 // the short version of jmpConUCF2 contains multiple branches,
2736 // making the reach slightly less
2737 if (rule == jmpConUCF2_rule)
2738 return (-126 <= offset && offset <= 125);
2739 return (-128 <= offset && offset <= 127);
2740 }
2741
2742 #ifdef ASSERT
2743 // Return whether or not this register is ever used as an argument.
2744 bool Matcher::can_be_java_arg(int reg)
2745 {
2746 return
2747 reg == RDI_num || reg == RDI_H_num ||
2748 reg == RSI_num || reg == RSI_H_num ||
2749 reg == RDX_num || reg == RDX_H_num ||
2750 reg == RCX_num || reg == RCX_H_num ||
2751 reg == R8_num || reg == R8_H_num ||
2752 reg == R9_num || reg == R9_H_num ||
2753 reg == R12_num || reg == R12_H_num ||
2754 reg == XMM0_num || reg == XMM0b_num ||
2755 reg == XMM1_num || reg == XMM1b_num ||
2756 reg == XMM2_num || reg == XMM2b_num ||
2757 reg == XMM3_num || reg == XMM3b_num ||
2758 reg == XMM4_num || reg == XMM4b_num ||
2759 reg == XMM5_num || reg == XMM5b_num ||
2760 reg == XMM6_num || reg == XMM6b_num ||
2761 reg == XMM7_num || reg == XMM7b_num;
2762 }
2763 #endif
2764
2765 uint Matcher::int_pressure_limit()
2766 {
2767 return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
2768 }
2769
2770 uint Matcher::float_pressure_limit()
2771 {
2772 // After experiment around with different values, the following default threshold
2773 // works best for LCM's register pressure scheduling on x64.
2774 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
2775 uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
2776 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
2777 }
2778
2779 // Register for the first projection of an int pair
2780 const RegMask& Matcher::firstI_proj_mask() {
2781 return INT_RAX_REG_mask();
2782 }
2783
2784 // Register for the second projection of an int pair
2785 const RegMask& Matcher::secondI_proj_mask() {
2786 return INT_RDX_REG_mask();
2787 }
2788
2789 // Register for the first projection of a long pair
2790 const RegMask& Matcher::firstL_proj_mask() {
2791 return LONG_RAX_REG_mask();
2792 }
2793
2794 // Register for the second projection of a long pair
2795 const RegMask& Matcher::secondL_proj_mask() {
2796 return LONG_RDX_REG_mask();
2797 }
2798
2799 %}
2800
2801 source_hpp %{
2802 // Header information of the source block.
2803 // Method declarations/definitions which are used outside
2804 // the ad-scope can conveniently be defined here.
2805 //
2806 // To keep related declarations/definitions/uses close together,
2807 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
2808
2809 #include "runtime/vm_version.hpp"
2810
2811 class NativeJump;
2812
2813 class CallStubImpl {
2814
2815 //--------------------------------------------------------------
2816 //---< Used for optimization in Compile::shorten_branches >---
2817 //--------------------------------------------------------------
2818
2819 public:
2820 // Size of call trampoline stub.
2821 static uint size_call_trampoline() {
2822 return 0; // no call trampolines on this platform
2823 }
2824
2825 // number of relocations needed by a call trampoline stub
2826 static uint reloc_call_trampoline() {
2827 return 0; // no call trampolines on this platform
2828 }
2829 };
2830
2831 class HandlerImpl {
2832
2833 public:
2834
2835 static int emit_deopt_handler(C2_MacroAssembler* masm);
2836
2837 static uint size_deopt_handler() {
2838 // one call and one jmp.
2839 return 7;
2840 }
2841 };
2842
2843 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
2844 switch(bytes) {
2845 case 4: // fall-through
2846 case 8: // fall-through
2847 case 16: return Assembler::AVX_128bit;
2848 case 32: return Assembler::AVX_256bit;
2849 case 64: return Assembler::AVX_512bit;
2850
2851 default: {
2852 ShouldNotReachHere();
2853 return Assembler::AVX_NoVec;
2854 }
2855 }
2856 }
2857
2858 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
2859 return vector_length_encoding(Matcher::vector_length_in_bytes(n));
2860 }
2861
2862 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
2863 uint def_idx = use->operand_index(opnd);
2864 Node* def = use->in(def_idx);
2865 return vector_length_encoding(def);
2866 }
2867
2868 static inline bool is_vector_popcount_predicate(BasicType bt) {
2869 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
2870 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
2871 }
2872
2873 static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_bytes) {
2874 return is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd() &&
2875 (VM_Version::supports_avx512vl() || vlen_bytes == 64);
2876 }
2877
2878 class Node::PD {
2879 public:
2880 enum NodeFlags : uint64_t {
2881 Flag_intel_jcc_erratum = Node::_last_flag << 1,
2882 Flag_sets_carry_flag = Node::_last_flag << 2,
2883 Flag_sets_parity_flag = Node::_last_flag << 3,
2884 Flag_sets_zero_flag = Node::_last_flag << 4,
2885 Flag_sets_overflow_flag = Node::_last_flag << 5,
2886 Flag_sets_sign_flag = Node::_last_flag << 6,
2887 Flag_clears_carry_flag = Node::_last_flag << 7,
2888 Flag_clears_parity_flag = Node::_last_flag << 8,
2889 Flag_clears_zero_flag = Node::_last_flag << 9,
2890 Flag_clears_overflow_flag = Node::_last_flag << 10,
2891 Flag_clears_sign_flag = Node::_last_flag << 11,
2892 Flag_ndd_demotable_opr1 = Node::_last_flag << 12,
2893 Flag_ndd_demotable_opr2 = Node::_last_flag << 13,
2894 _last_flag = Flag_ndd_demotable_opr2
2895 };
2896 };
2897
2898 %} // end source_hpp
2899
2900 source %{
2901
2902 #include "opto/addnode.hpp"
2903 #include "c2_intelJccErratum_x86.hpp"
2904
2905 void PhaseOutput::pd_perform_mach_node_analysis() {
2906 if (VM_Version::has_intel_jcc_erratum()) {
2907 int extra_padding = IntelJccErratum::tag_affected_machnodes(C, C->cfg(), C->regalloc());
2908 _buf_sizes._code += extra_padding;
2909 }
2910 }
2911
2912 int MachNode::pd_alignment_required() const {
2913 if (VM_Version::has_intel_jcc_erratum() && IntelJccErratum::is_jcc_erratum_branch(this)) {
2914 // Conservatively add worst case padding. We assume that relocInfo::addr_unit() is 1 on x86.
2915 return IntelJccErratum::largest_jcc_size() + 1;
2916 } else {
2917 return 1;
2918 }
2919 }
2920
2921 int MachNode::compute_padding(int current_offset) const {
2922 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
2923 Compile* C = Compile::current();
2924 PhaseOutput* output = C->output();
2925 Block* block = output->block();
2926 int index = output->index();
2927 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
2928 } else {
2929 return 0;
2930 }
2931 }
2932
2933 // Emit deopt handler code.
2934 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) {
2935
2936 // Note that the code buffer's insts_mark is always relative to insts.
2937 // That's why we must use the macroassembler to generate a handler.
2938 address base = __ start_a_stub(size_deopt_handler());
2939 if (base == nullptr) {
2940 ciEnv::current()->record_failure("CodeCache is full");
2941 return 0; // CodeBuffer::expand failed
2942 }
2943 int offset = __ offset();
2944
2945 Label start;
2946 __ bind(start);
2947
2948 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2949
2950 int entry_offset = __ offset();
2951
2952 __ jmp(start);
2953
2954 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
2955 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2956 "out of bounds read in post-call NOP check");
2957 __ end_a_stub();
2958 return entry_offset;
2959 }
2960
2961 static Assembler::Width widthForType(BasicType bt) {
2962 if (bt == T_BYTE) {
2963 return Assembler::B;
2964 } else if (bt == T_SHORT) {
2965 return Assembler::W;
2966 } else if (bt == T_INT) {
2967 return Assembler::D;
2968 } else {
2969 assert(bt == T_LONG, "not a long: %s", type2name(bt));
2970 return Assembler::Q;
2971 }
2972 }
2973
2974 //=============================================================================
2975
2976 // Float masks come from different places depending on platform.
2977 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
2978 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
2979 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
2980 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
2981 static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
2982 static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
2983 static address vector_byte_perm_mask() { return StubRoutines::x86::vector_byte_perm_mask(); }
2984 static address vector_long_sign_mask() { return StubRoutines::x86::vector_long_sign_mask(); }
2985 static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
2986 static address vector_int_mask_cmp_bits() { return StubRoutines::x86::vector_int_mask_cmp_bits(); }
2987 static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
2988 static address vector_byte_shufflemask() { return StubRoutines::x86::vector_byte_shuffle_mask(); }
2989 static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
2990 static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
2991 static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
2992 static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
2993 static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
2994 static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
2995 static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
2996
2997 //=============================================================================
2998 bool Matcher::match_rule_supported(int opcode) {
2999 if (!has_match_rule(opcode)) {
3000 return false; // no match rule present
3001 }
3002 switch (opcode) {
3003 case Op_AbsVL:
3004 case Op_StoreVectorScatter:
3005 if (UseAVX < 3) {
3006 return false;
3007 }
3008 break;
3009 case Op_PopCountI:
3010 case Op_PopCountL:
3011 if (!UsePopCountInstruction) {
3012 return false;
3013 }
3014 break;
3015 case Op_PopCountVI:
3016 if (UseAVX < 2) {
3017 return false;
3018 }
3019 break;
3020 case Op_CompressV:
3021 case Op_ExpandV:
3022 case Op_PopCountVL:
3023 if (UseAVX < 2) {
3024 return false;
3025 }
3026 break;
3027 case Op_MulVI:
3028 if ((UseSSE < 4) && (UseAVX < 1)) { // only with SSE4_1 or AVX
3029 return false;
3030 }
3031 break;
3032 case Op_MulVL:
3033 if (UseSSE < 4) { // only with SSE4_1 or AVX
3034 return false;
3035 }
3036 break;
3037 case Op_MulReductionVL:
3038 if (VM_Version::supports_avx512dq() == false) {
3039 return false;
3040 }
3041 break;
3042 case Op_AbsVB:
3043 case Op_AbsVS:
3044 case Op_AbsVI:
3045 case Op_AddReductionVI:
3046 case Op_AndReductionV:
3047 case Op_OrReductionV:
3048 case Op_XorReductionV:
3049 if (UseSSE < 3) { // requires at least SSSE3
3050 return false;
3051 }
3052 break;
3053 case Op_MaxHF:
3054 case Op_MinHF:
3055 if (!VM_Version::supports_avx512vlbw()) {
3056 return false;
3057 } // fallthrough
3058 case Op_AddHF:
3059 case Op_DivHF:
3060 case Op_FmaHF:
3061 case Op_MulHF:
3062 case Op_ReinterpretS2HF:
3063 case Op_ReinterpretHF2S:
3064 case Op_SubHF:
3065 case Op_SqrtHF:
3066 if (!VM_Version::supports_avx512_fp16()) {
3067 return false;
3068 }
3069 break;
3070 case Op_VectorLoadShuffle:
3071 case Op_VectorRearrange:
3072 case Op_MulReductionVI:
3073 if (UseSSE < 4) { // requires at least SSE4
3074 return false;
3075 }
3076 break;
3077 case Op_IsInfiniteF:
3078 case Op_IsInfiniteD:
3079 if (!VM_Version::supports_avx512dq()) {
3080 return false;
3081 }
3082 break;
3083 case Op_SqrtVD:
3084 case Op_SqrtVF:
3085 case Op_VectorMaskCmp:
3086 case Op_VectorCastB2X:
3087 case Op_VectorCastS2X:
3088 case Op_VectorCastI2X:
3089 case Op_VectorCastL2X:
3090 case Op_VectorCastF2X:
3091 case Op_VectorCastD2X:
3092 case Op_VectorUCastB2X:
3093 case Op_VectorUCastS2X:
3094 case Op_VectorUCastI2X:
3095 case Op_VectorMaskCast:
3096 if (UseAVX < 1) { // enabled for AVX only
3097 return false;
3098 }
3099 break;
3100 case Op_PopulateIndex:
3101 if (UseAVX < 2) {
3102 return false;
3103 }
3104 break;
3105 case Op_RoundVF:
3106 if (UseAVX < 2) { // enabled for AVX2 only
3107 return false;
3108 }
3109 break;
3110 case Op_RoundVD:
3111 if (UseAVX < 3) {
3112 return false; // enabled for AVX3 only
3113 }
3114 break;
3115 case Op_CompareAndSwapL:
3116 case Op_CompareAndSwapP:
3117 break;
3118 case Op_StrIndexOf:
3119 if (!UseSSE42Intrinsics) {
3120 return false;
3121 }
3122 break;
3123 case Op_StrIndexOfChar:
3124 if (!UseSSE42Intrinsics) {
3125 return false;
3126 }
3127 break;
3128 case Op_OnSpinWait:
3129 if (VM_Version::supports_on_spin_wait() == false) {
3130 return false;
3131 }
3132 break;
3133 case Op_MulVB:
3134 case Op_LShiftVB:
3135 case Op_RShiftVB:
3136 case Op_URShiftVB:
3137 case Op_VectorInsert:
3138 case Op_VectorLoadMask:
3139 case Op_VectorStoreMask:
3140 case Op_VectorBlend:
3141 if (UseSSE < 4) {
3142 return false;
3143 }
3144 break;
3145 case Op_MaxD:
3146 case Op_MaxF:
3147 case Op_MinD:
3148 case Op_MinF:
3149 if (UseAVX < 1) { // enabled for AVX only
3150 return false;
3151 }
3152 break;
3153 case Op_CacheWB:
3154 case Op_CacheWBPreSync:
3155 case Op_CacheWBPostSync:
3156 if (!VM_Version::supports_data_cache_line_flush()) {
3157 return false;
3158 }
3159 break;
3160 case Op_ExtractB:
3161 case Op_ExtractL:
3162 case Op_ExtractI:
3163 case Op_RoundDoubleMode:
3164 if (UseSSE < 4) {
3165 return false;
3166 }
3167 break;
3168 case Op_RoundDoubleModeV:
3169 if (VM_Version::supports_avx() == false) {
3170 return false; // 128bit vroundpd is not available
3171 }
3172 break;
3173 case Op_LoadVectorGather:
3174 case Op_LoadVectorGatherMasked:
3175 if (UseAVX < 2) {
3176 return false;
3177 }
3178 break;
3179 case Op_FmaF:
3180 case Op_FmaD:
3181 case Op_FmaVD:
3182 case Op_FmaVF:
3183 if (!UseFMA) {
3184 return false;
3185 }
3186 break;
3187 case Op_MacroLogicV:
3188 if (UseAVX < 3 || !UseVectorMacroLogic) {
3189 return false;
3190 }
3191 break;
3192
3193 case Op_VectorCmpMasked:
3194 if (UseAVX < 3 || !UseCountTrailingZerosInstruction) {
3195 return false;
3196 }
3197 break;
3198 case Op_VectorMaskGen:
3199 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3200 return false;
3201 }
3202 break;
3203 case Op_VectorMaskFirstTrue:
3204 case Op_VectorMaskLastTrue:
3205 case Op_VectorMaskTrueCount:
3206 case Op_VectorMaskToLong:
3207 if (UseAVX < 1) {
3208 return false;
3209 }
3210 break;
3211 case Op_RoundF:
3212 case Op_RoundD:
3213 break;
3214 case Op_CopySignD:
3215 case Op_CopySignF:
3216 if (UseAVX < 3) {
3217 return false;
3218 }
3219 if (!VM_Version::supports_avx512vl()) {
3220 return false;
3221 }
3222 break;
3223 case Op_CompressBits:
3224 case Op_ExpandBits:
3225 if (!VM_Version::supports_bmi2()) {
3226 return false;
3227 }
3228 break;
3229 case Op_CompressM:
3230 if (!VM_Version::supports_avx512vl() || !VM_Version::supports_bmi2()) {
3231 return false;
3232 }
3233 break;
3234 case Op_ConvF2HF:
3235 case Op_ConvHF2F:
3236 if (!VM_Version::supports_float16()) {
3237 return false;
3238 }
3239 break;
3240 case Op_VectorCastF2HF:
3241 case Op_VectorCastHF2F:
3242 if (!VM_Version::supports_f16c() && !VM_Version::supports_evex()) {
3243 return false;
3244 }
3245 break;
3246 }
3247 return true; // Match rules are supported by default.
3248 }
3249
3250 //------------------------------------------------------------------------
3251
3252 static inline bool is_pop_count_instr_target(BasicType bt) {
3253 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
3254 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
3255 }
3256
3257 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
3258 return match_rule_supported_vector(opcode, vlen, bt);
3259 }
3260
3261 // Identify extra cases that we might want to provide match rules for vector nodes and
3262 // other intrinsics guarded with vector length (vlen) and element type (bt).
3263 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
3264 if (!match_rule_supported(opcode)) {
3265 return false;
3266 }
3267 // Matcher::vector_size_supported() restricts vector sizes in the following way (see Matcher::vector_width_in_bytes):
3268 // * SSE2 supports 128bit vectors for all types;
3269 // * AVX1 supports 256bit vectors only for FLOAT and DOUBLE types;
3270 // * AVX2 supports 256bit vectors for all types;
3271 // * AVX512F supports 512bit vectors only for INT, FLOAT, and DOUBLE types;
3272 // * AVX512BW supports 512bit vectors for BYTE, SHORT, and CHAR types.
3273 // There's also a limit on minimum vector size supported: 2 elements (or 4 bytes for BYTE).
3274 // And MaxVectorSize is taken into account as well.
3275 if (!vector_size_supported(bt, vlen)) {
3276 return false;
3277 }
3278 // Special cases which require vector length follow:
3279 // * implementation limitations
3280 // * some 512bit vector operations on FLOAT and DOUBLE types require AVX512DQ
3281 // * 128bit vroundpd instruction is present only in AVX1
3282 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3283 switch (opcode) {
3284 case Op_MaxVHF:
3285 case Op_MinVHF:
3286 if (!VM_Version::supports_avx512bw()) {
3287 return false;
3288 }
3289 case Op_AddVHF:
3290 case Op_DivVHF:
3291 case Op_FmaVHF:
3292 case Op_MulVHF:
3293 case Op_SubVHF:
3294 case Op_SqrtVHF:
3295 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3296 return false;
3297 }
3298 if (!VM_Version::supports_avx512_fp16()) {
3299 return false;
3300 }
3301 break;
3302 case Op_AbsVF:
3303 case Op_NegVF:
3304 if ((vlen == 16) && (VM_Version::supports_avx512dq() == false)) {
3305 return false; // 512bit vandps and vxorps are not available
3306 }
3307 break;
3308 case Op_AbsVD:
3309 case Op_NegVD:
3310 if ((vlen == 8) && (VM_Version::supports_avx512dq() == false)) {
3311 return false; // 512bit vpmullq, vandpd and vxorpd are not available
3312 }
3313 break;
3314 case Op_RotateRightV:
3315 case Op_RotateLeftV:
3316 if (bt != T_INT && bt != T_LONG) {
3317 return false;
3318 } // fallthrough
3319 case Op_MacroLogicV:
3320 if (!VM_Version::supports_evex() ||
3321 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {
3322 return false;
3323 }
3324 break;
3325 case Op_ClearArray:
3326 case Op_VectorMaskGen:
3327 case Op_VectorCmpMasked:
3328 if (!VM_Version::supports_avx512bw()) {
3329 return false;
3330 }
3331 if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
3332 return false;
3333 }
3334 break;
3335 case Op_LoadVectorMasked:
3336 case Op_StoreVectorMasked:
3337 if (!VM_Version::supports_avx512bw() && (is_subword_type(bt) || UseAVX < 1)) {
3338 return false;
3339 }
3340 break;
3341 case Op_UMinV:
3342 case Op_UMaxV:
3343 if (UseAVX == 0) {
3344 return false;
3345 }
3346 break;
3347 case Op_UMinReductionV:
3348 case Op_UMaxReductionV:
3349 if (UseAVX == 0) {
3350 return false;
3351 }
3352 if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
3353 return false;
3354 }
3355 if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
3356 return false;
3357 }
3358 break;
3359 case Op_MaxV:
3360 case Op_MinV:
3361 if (UseSSE < 4 && is_integral_type(bt)) {
3362 return false;
3363 }
3364 if ((bt == T_FLOAT || bt == T_DOUBLE)) {
3365 // Float/Double intrinsics are enabled for AVX family currently.
3366 if (UseAVX == 0) {
3367 return false;
3368 }
3369 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) { // 512 bit Float/Double intrinsics need AVX512DQ
3370 return false;
3371 }
3372 }
3373 break;
3374 case Op_CallLeafVector:
3375 if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
3376 return false;
3377 }
3378 break;
3379 case Op_AddReductionVI:
3380 if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
3381 return false;
3382 }
3383 // fallthrough
3384 case Op_AndReductionV:
3385 case Op_OrReductionV:
3386 case Op_XorReductionV:
3387 if (is_subword_type(bt) && (UseSSE < 4)) {
3388 return false;
3389 }
3390 break;
3391 case Op_MinReductionV:
3392 case Op_MaxReductionV:
3393 if ((bt == T_INT || is_subword_type(bt)) && UseSSE < 4) {
3394 return false;
3395 } else if (bt == T_LONG && (UseAVX < 3 || !VM_Version::supports_avx512vlbwdq())) {
3396 return false;
3397 }
3398 // Float/Double intrinsics enabled for AVX family.
3399 if (UseAVX == 0 && (bt == T_FLOAT || bt == T_DOUBLE)) {
3400 return false;
3401 }
3402 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) {
3403 return false;
3404 }
3405 break;
3406 case Op_VectorBlend:
3407 if (UseAVX == 0 && size_in_bits < 128) {
3408 return false;
3409 }
3410 break;
3411 case Op_VectorTest:
3412 if (UseSSE < 4) {
3413 return false; // Implementation limitation
3414 } else if (size_in_bits < 32) {
3415 return false; // Implementation limitation
3416 }
3417 break;
3418 case Op_VectorLoadShuffle:
3419 case Op_VectorRearrange:
3420 if(vlen == 2) {
3421 return false; // Implementation limitation due to how shuffle is loaded
3422 } else if (size_in_bits == 256 && UseAVX < 2) {
3423 return false; // Implementation limitation
3424 }
3425 break;
3426 case Op_VectorLoadMask:
3427 case Op_VectorMaskCast:
3428 if (size_in_bits == 256 && UseAVX < 2) {
3429 return false; // Implementation limitation
3430 }
3431 // fallthrough
3432 case Op_VectorStoreMask:
3433 if (vlen == 2) {
3434 return false; // Implementation limitation
3435 }
3436 break;
3437 case Op_PopulateIndex:
3438 if (size_in_bits > 256 && !VM_Version::supports_avx512bw()) {
3439 return false;
3440 }
3441 break;
3442 case Op_VectorCastB2X:
3443 case Op_VectorCastS2X:
3444 case Op_VectorCastI2X:
3445 if (bt != T_DOUBLE && size_in_bits == 256 && UseAVX < 2) {
3446 return false;
3447 }
3448 break;
3449 case Op_VectorCastL2X:
3450 if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) {
3451 return false;
3452 } else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) {
3453 return false;
3454 }
3455 break;
3456 case Op_VectorCastF2X: {
3457 // As per JLS section 5.1.3 narrowing conversion to sub-word types
3458 // happen after intermediate conversion to integer and special handling
3459 // code needs AVX2 vpcmpeqd instruction for 256 bit vectors.
3460 int src_size_in_bits = type2aelembytes(T_FLOAT) * vlen * BitsPerByte;
3461 if (is_integral_type(bt) && src_size_in_bits == 256 && UseAVX < 2) {
3462 return false;
3463 }
3464 }
3465 // fallthrough
3466 case Op_VectorCastD2X:
3467 if (bt == T_LONG && !VM_Version::supports_avx512dq()) {
3468 return false;
3469 }
3470 break;
3471 case Op_VectorCastF2HF:
3472 case Op_VectorCastHF2F:
3473 if (!VM_Version::supports_f16c() &&
3474 ((!VM_Version::supports_evex() ||
3475 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())))) {
3476 return false;
3477 }
3478 break;
3479 case Op_RoundVD:
3480 if (!VM_Version::supports_avx512dq()) {
3481 return false;
3482 }
3483 break;
3484 case Op_MulReductionVI:
3485 if (bt == T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3486 return false;
3487 }
3488 break;
3489 case Op_LoadVectorGatherMasked:
3490 if (!is_subword_type(bt) && size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3491 return false;
3492 }
3493 if (is_subword_type(bt) &&
3494 ((size_in_bits > 256 && !VM_Version::supports_avx512bw()) ||
3495 (size_in_bits < 64) ||
3496 (bt == T_SHORT && !VM_Version::supports_bmi2()))) {
3497 return false;
3498 }
3499 break;
3500 case Op_StoreVectorScatterMasked:
3501 case Op_StoreVectorScatter:
3502 if (is_subword_type(bt)) {
3503 return false;
3504 } else if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3505 return false;
3506 }
3507 // fallthrough
3508 case Op_LoadVectorGather:
3509 if (!is_subword_type(bt) && size_in_bits == 64) {
3510 return false;
3511 }
3512 if (is_subword_type(bt) && size_in_bits < 64) {
3513 return false;
3514 }
3515 break;
3516 case Op_SaturatingAddV:
3517 case Op_SaturatingSubV:
3518 if (UseAVX < 1) {
3519 return false; // Implementation limitation
3520 }
3521 if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3522 return false;
3523 }
3524 break;
3525 case Op_SelectFromTwoVector:
3526 if (size_in_bits < 128) {
3527 return false;
3528 }
3529 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3530 return false;
3531 }
3532 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3533 return false;
3534 }
3535 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3536 return false;
3537 }
3538 if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
3539 return false;
3540 }
3541 break;
3542 case Op_MaskAll:
3543 if (!VM_Version::supports_evex()) {
3544 return false;
3545 }
3546 if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
3547 return false;
3548 }
3549 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3550 return false;
3551 }
3552 break;
3553 case Op_VectorMaskCmp:
3554 if (vlen < 2 || size_in_bits < 32) {
3555 return false;
3556 }
3557 break;
3558 case Op_CompressM:
3559 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3560 return false;
3561 }
3562 break;
3563 case Op_CompressV:
3564 case Op_ExpandV:
3565 if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
3566 return false;
3567 }
3568 if (size_in_bits < 128 ) {
3569 return false;
3570 }
3571 case Op_VectorLongToMask:
3572 if (UseAVX < 1) {
3573 return false;
3574 }
3575 if (UseAVX < 3 && !VM_Version::supports_bmi2()) {
3576 return false;
3577 }
3578 break;
3579 case Op_SignumVD:
3580 case Op_SignumVF:
3581 if (UseAVX < 1) {
3582 return false;
3583 }
3584 break;
3585 case Op_PopCountVI:
3586 case Op_PopCountVL: {
3587 if (!is_pop_count_instr_target(bt) &&
3588 (size_in_bits == 512) && !VM_Version::supports_avx512bw()) {
3589 return false;
3590 }
3591 }
3592 break;
3593 case Op_ReverseV:
3594 case Op_ReverseBytesV:
3595 if (UseAVX < 2) {
3596 return false;
3597 }
3598 break;
3599 case Op_CountTrailingZerosV:
3600 case Op_CountLeadingZerosV:
3601 if (UseAVX < 2) {
3602 return false;
3603 }
3604 break;
3605 }
3606 return true; // Per default match rules are supported.
3607 }
3608
3609 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
3610 // ADLC based match_rule_supported routine checks for the existence of pattern based
3611 // on IR opcode. Most of the unary/binary/ternary masked operation share the IR nodes
3612 // of their non-masked counterpart with mask edge being the differentiator.
3613 // This routine does a strict check on the existence of masked operation patterns
3614 // by returning a default false value for all the other opcodes apart from the
3615 // ones whose masked instruction patterns are defined in this file.
3616 if (!match_rule_supported_vector(opcode, vlen, bt)) {
3617 return false;
3618 }
3619
3620 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3621 if (size_in_bits != 512 && !VM_Version::supports_avx512vl()) {
3622 return false;
3623 }
3624 switch(opcode) {
3625 // Unary masked operations
3626 case Op_AbsVB:
3627 case Op_AbsVS:
3628 if(!VM_Version::supports_avx512bw()) {
3629 return false; // Implementation limitation
3630 }
3631 case Op_AbsVI:
3632 case Op_AbsVL:
3633 return true;
3634
3635 // Ternary masked operations
3636 case Op_FmaVF:
3637 case Op_FmaVD:
3638 return true;
3639
3640 case Op_MacroLogicV:
3641 if(bt != T_INT && bt != T_LONG) {
3642 return false;
3643 }
3644 return true;
3645
3646 // Binary masked operations
3647 case Op_AddVB:
3648 case Op_AddVS:
3649 case Op_SubVB:
3650 case Op_SubVS:
3651 case Op_MulVS:
3652 case Op_LShiftVS:
3653 case Op_RShiftVS:
3654 case Op_URShiftVS:
3655 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3656 if (!VM_Version::supports_avx512bw()) {
3657 return false; // Implementation limitation
3658 }
3659 return true;
3660
3661 case Op_MulVL:
3662 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3663 if (!VM_Version::supports_avx512dq()) {
3664 return false; // Implementation limitation
3665 }
3666 return true;
3667
3668 case Op_AndV:
3669 case Op_OrV:
3670 case Op_XorV:
3671 case Op_RotateRightV:
3672 case Op_RotateLeftV:
3673 if (bt != T_INT && bt != T_LONG) {
3674 return false; // Implementation limitation
3675 }
3676 return true;
3677
3678 case Op_VectorLoadMask:
3679 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3680 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3681 return false;
3682 }
3683 return true;
3684
3685 case Op_AddVI:
3686 case Op_AddVL:
3687 case Op_AddVF:
3688 case Op_AddVD:
3689 case Op_SubVI:
3690 case Op_SubVL:
3691 case Op_SubVF:
3692 case Op_SubVD:
3693 case Op_MulVI:
3694 case Op_MulVF:
3695 case Op_MulVD:
3696 case Op_DivVF:
3697 case Op_DivVD:
3698 case Op_SqrtVF:
3699 case Op_SqrtVD:
3700 case Op_LShiftVI:
3701 case Op_LShiftVL:
3702 case Op_RShiftVI:
3703 case Op_RShiftVL:
3704 case Op_URShiftVI:
3705 case Op_URShiftVL:
3706 case Op_LoadVectorMasked:
3707 case Op_StoreVectorMasked:
3708 case Op_LoadVectorGatherMasked:
3709 case Op_StoreVectorScatterMasked:
3710 return true;
3711
3712 case Op_UMinV:
3713 case Op_UMaxV:
3714 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3715 return false;
3716 } // fallthrough
3717 case Op_MaxV:
3718 case Op_MinV:
3719 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3720 return false; // Implementation limitation
3721 }
3722 if (is_floating_point_type(bt) && !VM_Version::supports_avx10_2()) {
3723 return false; // Implementation limitation
3724 }
3725 return true;
3726 case Op_SaturatingAddV:
3727 case Op_SaturatingSubV:
3728 if (!is_subword_type(bt)) {
3729 return false;
3730 }
3731 if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
3732 return false; // Implementation limitation
3733 }
3734 return true;
3735
3736 case Op_VectorMaskCmp:
3737 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3738 return false; // Implementation limitation
3739 }
3740 return true;
3741
3742 case Op_VectorRearrange:
3743 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3744 return false; // Implementation limitation
3745 }
3746 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3747 return false; // Implementation limitation
3748 } else if ((bt == T_INT || bt == T_FLOAT) && size_in_bits < 256) {
3749 return false; // Implementation limitation
3750 }
3751 return true;
3752
3753 // Binary Logical operations
3754 case Op_AndVMask:
3755 case Op_OrVMask:
3756 case Op_XorVMask:
3757 if (vlen > 16 && !VM_Version::supports_avx512bw()) {
3758 return false; // Implementation limitation
3759 }
3760 return true;
3761
3762 case Op_PopCountVI:
3763 case Op_PopCountVL:
3764 if (!is_pop_count_instr_target(bt)) {
3765 return false;
3766 }
3767 return true;
3768
3769 case Op_MaskAll:
3770 return true;
3771
3772 case Op_CountLeadingZerosV:
3773 if (is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd()) {
3774 return true;
3775 }
3776 default:
3777 return false;
3778 }
3779 }
3780
3781 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
3782 return false;
3783 }
3784
3785 // Return true if Vector::rearrange needs preparation of the shuffle argument
3786 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
3787 switch (elem_bt) {
3788 case T_BYTE: return false;
3789 case T_SHORT: return !VM_Version::supports_avx512bw();
3790 case T_INT: return !VM_Version::supports_avx();
3791 case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
3792 default:
3793 ShouldNotReachHere();
3794 return false;
3795 }
3796 }
3797
3798 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
3799 // Prefer predicate if the mask type is "TypePVectMask".
3800 return vt->isa_pvectmask() != nullptr;
3801 }
3802
3803 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
3804 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
3805 bool legacy = (generic_opnd->opcode() == LEGVEC);
3806 if (!VM_Version::supports_avx512vlbwdq() && // KNL
3807 is_temp && !legacy && (ideal_reg == Op_VecZ)) {
3808 // Conservatively specialize 512bit vec TEMP operands to legVecZ (zmm0-15) on KNL.
3809 return new legVecZOper();
3810 }
3811 if (legacy) {
3812 switch (ideal_reg) {
3813 case Op_VecS: return new legVecSOper();
3814 case Op_VecD: return new legVecDOper();
3815 case Op_VecX: return new legVecXOper();
3816 case Op_VecY: return new legVecYOper();
3817 case Op_VecZ: return new legVecZOper();
3818 }
3819 } else {
3820 switch (ideal_reg) {
3821 case Op_VecS: return new vecSOper();
3822 case Op_VecD: return new vecDOper();
3823 case Op_VecX: return new vecXOper();
3824 case Op_VecY: return new vecYOper();
3825 case Op_VecZ: return new vecZOper();
3826 }
3827 }
3828 ShouldNotReachHere();
3829 return nullptr;
3830 }
3831
3832 bool Matcher::is_reg2reg_move(MachNode* m) {
3833 switch (m->rule()) {
3834 case MoveVec2Leg_rule:
3835 case MoveLeg2Vec_rule:
3836 case MoveF2VL_rule:
3837 case MoveF2LEG_rule:
3838 case MoveVL2F_rule:
3839 case MoveLEG2F_rule:
3840 case MoveD2VL_rule:
3841 case MoveD2LEG_rule:
3842 case MoveVL2D_rule:
3843 case MoveLEG2D_rule:
3844 return true;
3845 default:
3846 return false;
3847 }
3848 }
3849
3850 bool Matcher::is_generic_vector(MachOper* opnd) {
3851 switch (opnd->opcode()) {
3852 case VEC:
3853 case LEGVEC:
3854 return true;
3855 default:
3856 return false;
3857 }
3858 }
3859
3860 //------------------------------------------------------------------------
3861
3862 const RegMask* Matcher::predicate_reg_mask(void) {
3863 return &_VECTMASK_REG_mask;
3864 }
3865
3866 // Max vector size in bytes. 0 if not supported.
3867 int Matcher::vector_width_in_bytes(BasicType bt) {
3868 assert(is_java_primitive(bt), "only primitive type vectors");
3869 // SSE2 supports 128bit vectors for all types.
3870 // AVX2 supports 256bit vectors for all types.
3871 // AVX2/EVEX supports 512bit vectors for all types.
3872 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
3873 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
3874 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
3875 size = (UseAVX > 2) ? 64 : 32;
3876 if (UseAVX > 2 && (bt == T_BYTE || bt == T_SHORT || bt == T_CHAR))
3877 size = (VM_Version::supports_avx512bw()) ? 64 : 32;
3878 // Use flag to limit vector size.
3879 size = MIN2(size,(int)MaxVectorSize);
3880 // Minimum 2 values in vector (or 4 for bytes).
3881 switch (bt) {
3882 case T_DOUBLE:
3883 case T_LONG:
3884 if (size < 16) return 0;
3885 break;
3886 case T_FLOAT:
3887 case T_INT:
3888 if (size < 8) return 0;
3889 break;
3890 case T_BOOLEAN:
3891 if (size < 4) return 0;
3892 break;
3893 case T_CHAR:
3894 if (size < 4) return 0;
3895 break;
3896 case T_BYTE:
3897 if (size < 4) return 0;
3898 break;
3899 case T_SHORT:
3900 if (size < 4) return 0;
3901 break;
3902 default:
3903 ShouldNotReachHere();
3904 }
3905 return size;
3906 }
3907
3908 // Limits on vector size (number of elements) loaded into vector.
3909 int Matcher::max_vector_size(const BasicType bt) {
3910 return vector_width_in_bytes(bt)/type2aelembytes(bt);
3911 }
3912 int Matcher::min_vector_size(const BasicType bt) {
3913 int max_size = max_vector_size(bt);
3914 // Min size which can be loaded into vector is 4 bytes.
3915 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
3916 // Support for calling svml double64 vectors
3917 if (bt == T_DOUBLE) {
3918 size = 1;
3919 }
3920 return MIN2(size,max_size);
3921 }
3922
3923 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
3924 // Limit the max vector size for auto vectorization to 256 bits (32 bytes)
3925 // by default on Cascade Lake
3926 if (VM_Version::is_default_intel_cascade_lake()) {
3927 return MIN2(Matcher::max_vector_size(bt), 32 / type2aelembytes(bt));
3928 }
3929 return Matcher::max_vector_size(bt);
3930 }
3931
3932 int Matcher::scalable_vector_reg_size(const BasicType bt) {
3933 return -1;
3934 }
3935
3936 // Vector ideal reg corresponding to specified size in bytes
3937 uint Matcher::vector_ideal_reg(int size) {
3938 assert(MaxVectorSize >= size, "");
3939 switch(size) {
3940 case 4: return Op_VecS;
3941 case 8: return Op_VecD;
3942 case 16: return Op_VecX;
3943 case 32: return Op_VecY;
3944 case 64: return Op_VecZ;
3945 }
3946 ShouldNotReachHere();
3947 return 0;
3948 }
3949
3950 // Check for shift by small constant as well
3951 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
3952 if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
3953 shift->in(2)->get_int() <= 3 &&
3954 // Are there other uses besides address expressions?
3955 !matcher->is_visited(shift)) {
3956 address_visited.set(shift->_idx); // Flag as address_visited
3957 mstack.push(shift->in(2), Matcher::Visit);
3958 Node *conv = shift->in(1);
3959 // Allow Matcher to match the rule which bypass
3960 // ConvI2L operation for an array index on LP64
3961 // if the index value is positive.
3962 if (conv->Opcode() == Op_ConvI2L &&
3963 conv->as_Type()->type()->is_long()->_lo >= 0 &&
3964 // Are there other uses besides address expressions?
3965 !matcher->is_visited(conv)) {
3966 address_visited.set(conv->_idx); // Flag as address_visited
3967 mstack.push(conv->in(1), Matcher::Pre_Visit);
3968 } else {
3969 mstack.push(conv, Matcher::Pre_Visit);
3970 }
3971 return true;
3972 }
3973 return false;
3974 }
3975
3976 // This function identifies sub-graphs in which a 'load' node is
3977 // input to two different nodes, and such that it can be matched
3978 // with BMI instructions like blsi, blsr, etc.
3979 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
3980 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
3981 // refers to the same node.
3982 //
3983 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
3984 // This is a temporary solution until we make DAGs expressible in ADL.
3985 template<typename ConType>
3986 class FusedPatternMatcher {
3987 Node* _op1_node;
3988 Node* _mop_node;
3989 int _con_op;
3990
3991 static int match_next(Node* n, int next_op, int next_op_idx) {
3992 if (n->in(1) == nullptr || n->in(2) == nullptr) {
3993 return -1;
3994 }
3995
3996 if (next_op_idx == -1) { // n is commutative, try rotations
3997 if (n->in(1)->Opcode() == next_op) {
3998 return 1;
3999 } else if (n->in(2)->Opcode() == next_op) {
4000 return 2;
4001 }
4002 } else {
4003 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
4004 if (n->in(next_op_idx)->Opcode() == next_op) {
4005 return next_op_idx;
4006 }
4007 }
4008 return -1;
4009 }
4010
4011 public:
4012 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
4013 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { }
4014
4015 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative
4016 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative
4017 typename ConType::NativeType con_value) {
4018 if (_op1_node->Opcode() != op1) {
4019 return false;
4020 }
4021 if (_mop_node->outcnt() > 2) {
4022 return false;
4023 }
4024 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx);
4025 if (op1_op2_idx == -1) {
4026 return false;
4027 }
4028 // Memory operation must be the other edge
4029 int op1_mop_idx = (op1_op2_idx & 1) + 1;
4030
4031 // Check that the mop node is really what we want
4032 if (_op1_node->in(op1_mop_idx) == _mop_node) {
4033 Node* op2_node = _op1_node->in(op1_op2_idx);
4034 if (op2_node->outcnt() > 1) {
4035 return false;
4036 }
4037 assert(op2_node->Opcode() == op2, "Should be");
4038 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx);
4039 if (op2_con_idx == -1) {
4040 return false;
4041 }
4042 // Memory operation must be the other edge
4043 int op2_mop_idx = (op2_con_idx & 1) + 1;
4044 // Check that the memory operation is the same node
4045 if (op2_node->in(op2_mop_idx) == _mop_node) {
4046 // Now check the constant
4047 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
4048 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
4049 return true;
4050 }
4051 }
4052 }
4053 return false;
4054 }
4055 };
4056
4057 static bool is_bmi_pattern(Node* n, Node* m) {
4058 assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
4059 if (n != nullptr && m != nullptr) {
4060 if (m->Opcode() == Op_LoadI) {
4061 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
4062 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
4063 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
4064 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
4065 } else if (m->Opcode() == Op_LoadL) {
4066 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
4067 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
4068 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
4069 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
4070 }
4071 }
4072 return false;
4073 }
4074
4075 // Should the matcher clone input 'm' of node 'n'?
4076 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
4077 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
4078 if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
4079 mstack.push(m, Visit);
4080 return true;
4081 }
4082 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
4083 mstack.push(m, Visit); // m = ShiftCntV
4084 return true;
4085 }
4086 if (is_encode_and_store_pattern(n, m)) {
4087 mstack.push(m, Visit);
4088 return true;
4089 }
4090 return false;
4091 }
4092
4093 // Should the Matcher clone shifts on addressing modes, expecting them
4094 // to be subsumed into complex addressing expressions or compute them
4095 // into registers?
4096 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
4097 Node *off = m->in(AddPNode::Offset);
4098 if (off->is_Con()) {
4099 address_visited.test_set(m->_idx); // Flag as address_visited
4100 Node *adr = m->in(AddPNode::Address);
4101
4102 // Intel can handle 2 adds in addressing mode, with one of them using an immediate offset.
4103 // AtomicAdd is not an addressing expression.
4104 // Cheap to find it by looking for screwy base.
4105 if (adr->is_AddP() &&
4106 !adr->in(AddPNode::Base)->is_top() &&
4107 !adr->in(AddPNode::Offset)->is_Con() &&
4108 off->get_long() == (int) (off->get_long()) && // immL32
4109 // Are there other uses besides address expressions?
4110 !is_visited(adr)) {
4111 address_visited.set(adr->_idx); // Flag as address_visited
4112 Node *shift = adr->in(AddPNode::Offset);
4113 if (!clone_shift(shift, this, mstack, address_visited)) {
4114 mstack.push(shift, Pre_Visit);
4115 }
4116 mstack.push(adr->in(AddPNode::Address), Pre_Visit);
4117 mstack.push(adr->in(AddPNode::Base), Pre_Visit);
4118 } else {
4119 mstack.push(adr, Pre_Visit);
4120 }
4121
4122 // Clone X+offset as it also folds into most addressing expressions
4123 mstack.push(off, Visit);
4124 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4125 return true;
4126 } else if (clone_shift(off, this, mstack, address_visited)) {
4127 address_visited.test_set(m->_idx); // Flag as address_visited
4128 mstack.push(m->in(AddPNode::Address), Pre_Visit);
4129 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4130 return true;
4131 }
4132 return false;
4133 }
4134
4135 static inline Assembler::ComparisonPredicate booltest_pred_to_comparison_pred(int bt) {
4136 switch (bt) {
4137 case BoolTest::eq:
4138 return Assembler::eq;
4139 case BoolTest::ne:
4140 return Assembler::neq;
4141 case BoolTest::le:
4142 case BoolTest::ule:
4143 return Assembler::le;
4144 case BoolTest::ge:
4145 case BoolTest::uge:
4146 return Assembler::nlt;
4147 case BoolTest::lt:
4148 case BoolTest::ult:
4149 return Assembler::lt;
4150 case BoolTest::gt:
4151 case BoolTest::ugt:
4152 return Assembler::nle;
4153 default : ShouldNotReachHere(); return Assembler::_false;
4154 }
4155 }
4156
4157 static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_fp(int bt) {
4158 switch (bt) {
4159 case BoolTest::eq: return Assembler::EQ_OQ; // ordered non-signaling
4160 // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
4161 case BoolTest::ne: return Assembler::NEQ_UQ; // unordered non-signaling
4162 case BoolTest::le: return Assembler::LE_OQ; // ordered non-signaling
4163 case BoolTest::ge: return Assembler::GE_OQ; // ordered non-signaling
4164 case BoolTest::lt: return Assembler::LT_OQ; // ordered non-signaling
4165 case BoolTest::gt: return Assembler::GT_OQ; // ordered non-signaling
4166 default: ShouldNotReachHere(); return Assembler::FALSE_OS;
4167 }
4168 }
4169
4170 // Helper methods for MachSpillCopyNode::implementation().
4171 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
4172 int src_hi, int dst_hi, uint ireg, outputStream* st) {
4173 assert(ireg == Op_VecS || // 32bit vector
4174 ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
4175 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
4176 "no non-adjacent vector moves" );
4177 if (masm) {
4178 switch (ireg) {
4179 case Op_VecS: // copy whole register
4180 case Op_VecD:
4181 case Op_VecX:
4182 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4183 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4184 } else {
4185 __ vextractf32x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4186 }
4187 break;
4188 case Op_VecY:
4189 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4190 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4191 } else {
4192 __ vextractf64x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4193 }
4194 break;
4195 case Op_VecZ:
4196 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
4197 break;
4198 default:
4199 ShouldNotReachHere();
4200 }
4201 #ifndef PRODUCT
4202 } else {
4203 switch (ireg) {
4204 case Op_VecS:
4205 case Op_VecD:
4206 case Op_VecX:
4207 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4208 break;
4209 case Op_VecY:
4210 case Op_VecZ:
4211 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4212 break;
4213 default:
4214 ShouldNotReachHere();
4215 }
4216 #endif
4217 }
4218 }
4219
4220 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
4221 int stack_offset, int reg, uint ireg, outputStream* st) {
4222 if (masm) {
4223 if (is_load) {
4224 switch (ireg) {
4225 case Op_VecS:
4226 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4227 break;
4228 case Op_VecD:
4229 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4230 break;
4231 case Op_VecX:
4232 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4233 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4234 } else {
4235 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4236 __ vinsertf32x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4237 }
4238 break;
4239 case Op_VecY:
4240 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4241 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4242 } else {
4243 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4244 __ vinsertf64x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4245 }
4246 break;
4247 case Op_VecZ:
4248 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
4249 break;
4250 default:
4251 ShouldNotReachHere();
4252 }
4253 } else { // store
4254 switch (ireg) {
4255 case Op_VecS:
4256 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4257 break;
4258 case Op_VecD:
4259 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4260 break;
4261 case Op_VecX:
4262 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4263 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4264 }
4265 else {
4266 __ vextractf32x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4267 }
4268 break;
4269 case Op_VecY:
4270 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4271 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4272 }
4273 else {
4274 __ vextractf64x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4275 }
4276 break;
4277 case Op_VecZ:
4278 __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4279 break;
4280 default:
4281 ShouldNotReachHere();
4282 }
4283 }
4284 #ifndef PRODUCT
4285 } else {
4286 if (is_load) {
4287 switch (ireg) {
4288 case Op_VecS:
4289 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4290 break;
4291 case Op_VecD:
4292 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4293 break;
4294 case Op_VecX:
4295 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4296 break;
4297 case Op_VecY:
4298 case Op_VecZ:
4299 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4300 break;
4301 default:
4302 ShouldNotReachHere();
4303 }
4304 } else { // store
4305 switch (ireg) {
4306 case Op_VecS:
4307 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4308 break;
4309 case Op_VecD:
4310 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4311 break;
4312 case Op_VecX:
4313 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4314 break;
4315 case Op_VecY:
4316 case Op_VecZ:
4317 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4318 break;
4319 default:
4320 ShouldNotReachHere();
4321 }
4322 }
4323 #endif
4324 }
4325 }
4326
4327 template <class T>
4328 static inline GrowableArray<jbyte>* vreplicate_imm(BasicType bt, T con, int len) {
4329 int size = type2aelembytes(bt) * len;
4330 GrowableArray<jbyte>* val = new GrowableArray<jbyte>(size, size, 0);
4331 for (int i = 0; i < len; i++) {
4332 int offset = i * type2aelembytes(bt);
4333 switch (bt) {
4334 case T_BYTE: val->at(i) = con; break;
4335 case T_SHORT: {
4336 jshort c = con;
4337 memcpy(val->adr_at(offset), &c, sizeof(jshort));
4338 break;
4339 }
4340 case T_INT: {
4341 jint c = con;
4342 memcpy(val->adr_at(offset), &c, sizeof(jint));
4343 break;
4344 }
4345 case T_LONG: {
4346 jlong c = con;
4347 memcpy(val->adr_at(offset), &c, sizeof(jlong));
4348 break;
4349 }
4350 case T_FLOAT: {
4351 jfloat c = con;
4352 memcpy(val->adr_at(offset), &c, sizeof(jfloat));
4353 break;
4354 }
4355 case T_DOUBLE: {
4356 jdouble c = con;
4357 memcpy(val->adr_at(offset), &c, sizeof(jdouble));
4358 break;
4359 }
4360 default: assert(false, "%s", type2name(bt));
4361 }
4362 }
4363 return val;
4364 }
4365
4366 static inline jlong high_bit_set(BasicType bt) {
4367 switch (bt) {
4368 case T_BYTE: return 0x8080808080808080;
4369 case T_SHORT: return 0x8000800080008000;
4370 case T_INT: return 0x8000000080000000;
4371 case T_LONG: return 0x8000000000000000;
4372 default:
4373 ShouldNotReachHere();
4374 return 0;
4375 }
4376 }
4377
4378 #ifndef PRODUCT
4379 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
4380 st->print("nop \t# %d bytes pad for loops and calls", _count);
4381 }
4382 #endif
4383
4384 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
4385 __ nop(_count);
4386 }
4387
4388 uint MachNopNode::size(PhaseRegAlloc*) const {
4389 return _count;
4390 }
4391
4392 #ifndef PRODUCT
4393 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
4394 st->print("# breakpoint");
4395 }
4396 #endif
4397
4398 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const {
4399 __ int3();
4400 }
4401
4402 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
4403 return MachNode::size(ra_);
4404 }
4405
4406 %}
4407
4408 //----------ENCODING BLOCK-----------------------------------------------------
4409 // This block specifies the encoding classes used by the compiler to
4410 // output byte streams. Encoding classes are parameterized macros
4411 // used by Machine Instruction Nodes in order to generate the bit
4412 // encoding of the instruction. Operands specify their base encoding
4413 // interface with the interface keyword. There are currently
4414 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
4415 // COND_INTER. REG_INTER causes an operand to generate a function
4416 // which returns its register number when queried. CONST_INTER causes
4417 // an operand to generate a function which returns the value of the
4418 // constant when queried. MEMORY_INTER causes an operand to generate
4419 // four functions which return the Base Register, the Index Register,
4420 // the Scale Value, and the Offset Value of the operand when queried.
4421 // COND_INTER causes an operand to generate six functions which return
4422 // the encoding code (ie - encoding bits for the instruction)
4423 // associated with each basic boolean condition for a conditional
4424 // instruction.
4425 //
4426 // Instructions specify two basic values for encoding. Again, a
4427 // function is available to check if the constant displacement is an
4428 // oop. They use the ins_encode keyword to specify their encoding
4429 // classes (which must be a sequence of enc_class names, and their
4430 // parameters, specified in the encoding block), and they use the
4431 // opcode keyword to specify, in order, their primary, secondary, and
4432 // tertiary opcode. Only the opcode sections which a particular
4433 // instruction needs for encoding need to be specified.
4434 encode %{
4435 enc_class cdql_enc(no_rax_rdx_RegI div)
4436 %{
4437 // Full implementation of Java idiv and irem; checks for
4438 // special case as described in JVM spec., p.243 & p.271.
4439 //
4440 // normal case special case
4441 //
4442 // input : rax: dividend min_int
4443 // reg: divisor -1
4444 //
4445 // output: rax: quotient (= rax idiv reg) min_int
4446 // rdx: remainder (= rax irem reg) 0
4447 //
4448 // Code sequnce:
4449 //
4450 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
4451 // 5: 75 07/08 jne e <normal>
4452 // 7: 33 d2 xor %edx,%edx
4453 // [div >= 8 -> offset + 1]
4454 // [REX_B]
4455 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
4456 // c: 74 03/04 je 11 <done>
4457 // 000000000000000e <normal>:
4458 // e: 99 cltd
4459 // [div >= 8 -> offset + 1]
4460 // [REX_B]
4461 // f: f7 f9 idiv $div
4462 // 0000000000000011 <done>:
4463 Label normal;
4464 Label done;
4465
4466 // cmp $0x80000000,%eax
4467 __ cmpl(as_Register(RAX_enc), 0x80000000);
4468
4469 // jne e <normal>
4470 __ jccb(Assembler::notEqual, normal);
4471
4472 // xor %edx,%edx
4473 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4474
4475 // cmp $0xffffffffffffffff,%ecx
4476 __ cmpl($div$$Register, -1);
4477
4478 // je 11 <done>
4479 __ jccb(Assembler::equal, done);
4480
4481 // <normal>
4482 // cltd
4483 __ bind(normal);
4484 __ cdql();
4485
4486 // idivl
4487 // <done>
4488 __ idivl($div$$Register);
4489 __ bind(done);
4490 %}
4491
4492 enc_class cdqq_enc(no_rax_rdx_RegL div)
4493 %{
4494 // Full implementation of Java ldiv and lrem; checks for
4495 // special case as described in JVM spec., p.243 & p.271.
4496 //
4497 // normal case special case
4498 //
4499 // input : rax: dividend min_long
4500 // reg: divisor -1
4501 //
4502 // output: rax: quotient (= rax idiv reg) min_long
4503 // rdx: remainder (= rax irem reg) 0
4504 //
4505 // Code sequnce:
4506 //
4507 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
4508 // 7: 00 00 80
4509 // a: 48 39 d0 cmp %rdx,%rax
4510 // d: 75 08 jne 17 <normal>
4511 // f: 33 d2 xor %edx,%edx
4512 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
4513 // 15: 74 05 je 1c <done>
4514 // 0000000000000017 <normal>:
4515 // 17: 48 99 cqto
4516 // 19: 48 f7 f9 idiv $div
4517 // 000000000000001c <done>:
4518 Label normal;
4519 Label done;
4520
4521 // mov $0x8000000000000000,%rdx
4522 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
4523
4524 // cmp %rdx,%rax
4525 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
4526
4527 // jne 17 <normal>
4528 __ jccb(Assembler::notEqual, normal);
4529
4530 // xor %edx,%edx
4531 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4532
4533 // cmp $0xffffffffffffffff,$div
4534 __ cmpq($div$$Register, -1);
4535
4536 // je 1e <done>
4537 __ jccb(Assembler::equal, done);
4538
4539 // <normal>
4540 // cqto
4541 __ bind(normal);
4542 __ cdqq();
4543
4544 // idivq (note: must be emitted by the user of this rule)
4545 // <done>
4546 __ idivq($div$$Register);
4547 __ bind(done);
4548 %}
4549
4550 enc_class clear_avx %{
4551 DEBUG_ONLY(int off0 = __ offset());
4552 if (generate_vzeroupper(Compile::current())) {
4553 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
4554 // Clear upper bits of YMM registers when current compiled code uses
4555 // wide vectors to avoid AVX <-> SSE transition penalty during call.
4556 __ vzeroupper();
4557 }
4558 DEBUG_ONLY(int off1 = __ offset());
4559 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
4560 %}
4561
4562 enc_class Java_To_Runtime(method meth) %{
4563 __ lea(r10, RuntimeAddress((address)$meth$$method));
4564 __ call(r10);
4565 __ post_call_nop();
4566 %}
4567
4568 enc_class Java_Static_Call(method meth)
4569 %{
4570 // JAVA STATIC CALL
4571 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
4572 // determine who we intended to call.
4573 if (!_method) {
4574 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
4575 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
4576 // The NOP here is purely to ensure that eliding a call to
4577 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
4578 __ nop(5);
4579 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
4580 } else {
4581 int method_index = resolved_method_index(masm);
4582 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4583 : static_call_Relocation::spec(method_index);
4584 address mark = __ pc();
4585 int call_offset = __ offset();
4586 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
4587 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
4588 // Calls of the same statically bound method can share
4589 // a stub to the interpreter.
4590 __ code()->shared_stub_to_interp_for(_method, call_offset);
4591 } else {
4592 // Emit stubs for static call.
4593 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
4594 __ clear_inst_mark();
4595 if (stub == nullptr) {
4596 ciEnv::current()->record_failure("CodeCache is full");
4597 return;
4598 }
4599 }
4600 }
4601 __ post_call_nop();
4602 %}
4603
4604 enc_class Java_Dynamic_Call(method meth) %{
4605 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4606 __ post_call_nop();
4607 %}
4608
4609 enc_class call_epilog %{
4610 if (VerifyStackAtCalls) {
4611 // Check that stack depth is unchanged: find majik cookie on stack
4612 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4613 Label L;
4614 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4615 __ jccb(Assembler::equal, L);
4616 // Die if stack mismatch
4617 __ int3();
4618 __ bind(L);
4619 }
4620 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
4621 // The last return value is not set by the callee but used to pass the null marker to compiled code.
4622 // Search for the corresponding projection, get the register and emit code that initializes it.
4623 uint con = (tf()->range_cc()->cnt() - 1);
4624 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
4625 ProjNode* proj = fast_out(i)->as_Proj();
4626 if (proj->_con == con) {
4627 // Set null marker if rax is non-null (a non-null value is returned buffered or scalarized)
4628 OptoReg::Name optoReg = ra_->get_reg_first(proj);
4629 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
4630 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
4631 __ testq(rax, rax);
4632 __ setb(Assembler::notZero, toReg);
4633 __ movzbl(toReg, toReg);
4634 if (reg->is_stack()) {
4635 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
4636 __ movq(Address(rsp, st_off), toReg);
4637 }
4638 break;
4639 }
4640 }
4641 if (return_value_is_used()) {
4642 // An inline type is returned as fields in multiple registers.
4643 // Rax either contains an oop if the inline type is buffered or a pointer
4644 // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
4645 // if the lowest bit is set to allow C2 to use the oop after null checking.
4646 // rax &= (rax & 1) - 1
4647 __ movptr(rscratch1, rax);
4648 __ andptr(rscratch1, 0x1);
4649 __ subptr(rscratch1, 0x1);
4650 __ andptr(rax, rscratch1);
4651 }
4652 }
4653 %}
4654
4655 %}
4656
4657 //----------FRAME--------------------------------------------------------------
4658 // Definition of frame structure and management information.
4659 //
4660 // S T A C K L A Y O U T Allocators stack-slot number
4661 // | (to get allocators register number
4662 // G Owned by | | v add OptoReg::stack0())
4663 // r CALLER | |
4664 // o | +--------+ pad to even-align allocators stack-slot
4665 // w V | pad0 | numbers; owned by CALLER
4666 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4667 // h ^ | in | 5
4668 // | | args | 4 Holes in incoming args owned by SELF
4669 // | | | | 3
4670 // | | +--------+
4671 // V | | old out| Empty on Intel, window on Sparc
4672 // | old |preserve| Must be even aligned.
4673 // | SP-+--------+----> Matcher::_old_SP, even aligned
4674 // | | in | 3 area for Intel ret address
4675 // Owned by |preserve| Empty on Sparc.
4676 // SELF +--------+
4677 // | | pad2 | 2 pad to align old SP
4678 // | +--------+ 1
4679 // | | locks | 0
4680 // | +--------+----> OptoReg::stack0(), even aligned
4681 // | | pad1 | 11 pad to align new SP
4682 // | +--------+
4683 // | | | 10
4684 // | | spills | 9 spills
4685 // V | | 8 (pad0 slot for callee)
4686 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4687 // ^ | out | 7
4688 // | | args | 6 Holes in outgoing args owned by CALLEE
4689 // Owned by +--------+
4690 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4691 // | new |preserve| Must be even-aligned.
4692 // | SP-+--------+----> Matcher::_new_SP, even aligned
4693 // | | |
4694 //
4695 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4696 // known from SELF's arguments and the Java calling convention.
4697 // Region 6-7 is determined per call site.
4698 // Note 2: If the calling convention leaves holes in the incoming argument
4699 // area, those holes are owned by SELF. Holes in the outgoing area
4700 // are owned by the CALLEE. Holes should not be necessary in the
4701 // incoming area, as the Java calling convention is completely under
4702 // the control of the AD file. Doubles can be sorted and packed to
4703 // avoid holes. Holes in the outgoing arguments may be necessary for
4704 // varargs C calling conventions.
4705 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4706 // even aligned with pad0 as needed.
4707 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4708 // region 6-11 is even aligned; it may be padded out more so that
4709 // the region from SP to FP meets the minimum stack alignment.
4710 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4711 // alignment. Region 11, pad1, may be dynamically extended so that
4712 // SP meets the minimum alignment.
4713
4714 frame
4715 %{
4716 // These three registers define part of the calling convention
4717 // between compiled code and the interpreter.
4718 inline_cache_reg(RAX); // Inline Cache Register
4719
4720 // Optional: name the operand used by cisc-spilling to access
4721 // [stack_pointer + offset]
4722 cisc_spilling_operand_name(indOffset32);
4723
4724 // Number of stack slots consumed by locking an object
4725 sync_stack_slots(2);
4726
4727 // Compiled code's Frame Pointer
4728 frame_pointer(RSP);
4729
4730 // Stack alignment requirement
4731 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4732
4733 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4734 // for calls to C. Supports the var-args backing area for register parms.
4735 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4736
4737 // The after-PROLOG location of the return address. Location of
4738 // return address specifies a type (REG or STACK) and a number
4739 // representing the register number (i.e. - use a register name) or
4740 // stack slot.
4741 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4742 // Otherwise, it is above the locks and verification slot and alignment word
4743 return_addr(STACK - 2 +
4744 align_up((Compile::current()->in_preserve_stack_slots() +
4745 Compile::current()->fixed_slots()),
4746 stack_alignment_in_slots()));
4747
4748 // Location of compiled Java return values. Same as C for now.
4749 return_value
4750 %{
4751 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4752 "only return normal values");
4753
4754 static const int lo[Op_RegL + 1] = {
4755 0,
4756 0,
4757 RAX_num, // Op_RegN
4758 RAX_num, // Op_RegI
4759 RAX_num, // Op_RegP
4760 XMM0_num, // Op_RegF
4761 XMM0_num, // Op_RegD
4762 RAX_num // Op_RegL
4763 };
4764 static const int hi[Op_RegL + 1] = {
4765 0,
4766 0,
4767 OptoReg::Bad, // Op_RegN
4768 OptoReg::Bad, // Op_RegI
4769 RAX_H_num, // Op_RegP
4770 OptoReg::Bad, // Op_RegF
4771 XMM0b_num, // Op_RegD
4772 RAX_H_num // Op_RegL
4773 };
4774 // Excluded flags and vector registers.
4775 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
4776 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4777 %}
4778 %}
4779
4780 //----------ATTRIBUTES---------------------------------------------------------
4781 //----------Operand Attributes-------------------------------------------------
4782 op_attrib op_cost(0); // Required cost attribute
4783
4784 //----------Instruction Attributes---------------------------------------------
4785 ins_attrib ins_cost(100); // Required cost attribute
4786 ins_attrib ins_size(8); // Required size attribute (in bits)
4787 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4788 // a non-matching short branch variant
4789 // of some long branch?
4790 ins_attrib ins_alignment(1); // Required alignment attribute (must
4791 // be a power of 2) specifies the
4792 // alignment that some part of the
4793 // instruction (not necessarily the
4794 // start) requires. If > 1, a
4795 // compute_padding() function must be
4796 // provided for the instruction
4797
4798 // Whether this node is expanded during code emission into a sequence of
4799 // instructions and the first instruction can perform an implicit null check.
4800 ins_attrib ins_is_late_expanded_null_check_candidate(false);
4801
4802 //----------OPERANDS-----------------------------------------------------------
4803 // Operand definitions must precede instruction definitions for correct parsing
4804 // in the ADLC because operands constitute user defined types which are used in
4805 // instruction definitions.
4806
4807 //----------Simple Operands----------------------------------------------------
4808 // Immediate Operands
4809 // Integer Immediate
4810 operand immI()
4811 %{
4812 match(ConI);
4813
4814 op_cost(10);
4815 format %{ %}
4816 interface(CONST_INTER);
4817 %}
4818
4819 // Constant for test vs zero
4820 operand immI_0()
4821 %{
4822 predicate(n->get_int() == 0);
4823 match(ConI);
4824
4825 op_cost(0);
4826 format %{ %}
4827 interface(CONST_INTER);
4828 %}
4829
4830 // Constant for increment
4831 operand immI_1()
4832 %{
4833 predicate(n->get_int() == 1);
4834 match(ConI);
4835
4836 op_cost(0);
4837 format %{ %}
4838 interface(CONST_INTER);
4839 %}
4840
4841 // Constant for decrement
4842 operand immI_M1()
4843 %{
4844 predicate(n->get_int() == -1);
4845 match(ConI);
4846
4847 op_cost(0);
4848 format %{ %}
4849 interface(CONST_INTER);
4850 %}
4851
4852 operand immI_2()
4853 %{
4854 predicate(n->get_int() == 2);
4855 match(ConI);
4856
4857 op_cost(0);
4858 format %{ %}
4859 interface(CONST_INTER);
4860 %}
4861
4862 operand immI_4()
4863 %{
4864 predicate(n->get_int() == 4);
4865 match(ConI);
4866
4867 op_cost(0);
4868 format %{ %}
4869 interface(CONST_INTER);
4870 %}
4871
4872 operand immI_8()
4873 %{
4874 predicate(n->get_int() == 8);
4875 match(ConI);
4876
4877 op_cost(0);
4878 format %{ %}
4879 interface(CONST_INTER);
4880 %}
4881
4882 // Valid scale values for addressing modes
4883 operand immI2()
4884 %{
4885 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4886 match(ConI);
4887
4888 format %{ %}
4889 interface(CONST_INTER);
4890 %}
4891
4892 operand immU7()
4893 %{
4894 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
4895 match(ConI);
4896
4897 op_cost(5);
4898 format %{ %}
4899 interface(CONST_INTER);
4900 %}
4901
4902 operand immI8()
4903 %{
4904 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4905 match(ConI);
4906
4907 op_cost(5);
4908 format %{ %}
4909 interface(CONST_INTER);
4910 %}
4911
4912 operand immU8()
4913 %{
4914 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
4915 match(ConI);
4916
4917 op_cost(5);
4918 format %{ %}
4919 interface(CONST_INTER);
4920 %}
4921
4922 operand immI16()
4923 %{
4924 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4925 match(ConI);
4926
4927 op_cost(10);
4928 format %{ %}
4929 interface(CONST_INTER);
4930 %}
4931
4932 // Int Immediate non-negative
4933 operand immU31()
4934 %{
4935 predicate(n->get_int() >= 0);
4936 match(ConI);
4937
4938 op_cost(0);
4939 format %{ %}
4940 interface(CONST_INTER);
4941 %}
4942
4943 // Pointer Immediate
4944 operand immP()
4945 %{
4946 match(ConP);
4947
4948 op_cost(10);
4949 format %{ %}
4950 interface(CONST_INTER);
4951 %}
4952
4953 // Null Pointer Immediate
4954 operand immP0()
4955 %{
4956 predicate(n->get_ptr() == 0);
4957 match(ConP);
4958
4959 op_cost(5);
4960 format %{ %}
4961 interface(CONST_INTER);
4962 %}
4963
4964 // Pointer Immediate
4965 operand immN() %{
4966 match(ConN);
4967
4968 op_cost(10);
4969 format %{ %}
4970 interface(CONST_INTER);
4971 %}
4972
4973 operand immNKlass() %{
4974 match(ConNKlass);
4975
4976 op_cost(10);
4977 format %{ %}
4978 interface(CONST_INTER);
4979 %}
4980
4981 // Null Pointer Immediate
4982 operand immN0() %{
4983 predicate(n->get_narrowcon() == 0);
4984 match(ConN);
4985
4986 op_cost(5);
4987 format %{ %}
4988 interface(CONST_INTER);
4989 %}
4990
4991 operand immP31()
4992 %{
4993 predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
4994 && (n->get_ptr() >> 31) == 0);
4995 match(ConP);
4996
4997 op_cost(5);
4998 format %{ %}
4999 interface(CONST_INTER);
5000 %}
5001
5002
5003 // Long Immediate
5004 operand immL()
5005 %{
5006 match(ConL);
5007
5008 op_cost(20);
5009 format %{ %}
5010 interface(CONST_INTER);
5011 %}
5012
5013 // Long Immediate 8-bit
5014 operand immL8()
5015 %{
5016 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
5017 match(ConL);
5018
5019 op_cost(5);
5020 format %{ %}
5021 interface(CONST_INTER);
5022 %}
5023
5024 // Long Immediate 32-bit unsigned
5025 operand immUL32()
5026 %{
5027 predicate(n->get_long() == (unsigned int) (n->get_long()));
5028 match(ConL);
5029
5030 op_cost(10);
5031 format %{ %}
5032 interface(CONST_INTER);
5033 %}
5034
5035 // Long Immediate 32-bit signed
5036 operand immL32()
5037 %{
5038 predicate(n->get_long() == (int) (n->get_long()));
5039 match(ConL);
5040
5041 op_cost(15);
5042 format %{ %}
5043 interface(CONST_INTER);
5044 %}
5045
5046 operand immL_Pow2()
5047 %{
5048 predicate(is_power_of_2((julong)n->get_long()));
5049 match(ConL);
5050
5051 op_cost(15);
5052 format %{ %}
5053 interface(CONST_INTER);
5054 %}
5055
5056 operand immL_NotPow2()
5057 %{
5058 predicate(is_power_of_2((julong)~n->get_long()));
5059 match(ConL);
5060
5061 op_cost(15);
5062 format %{ %}
5063 interface(CONST_INTER);
5064 %}
5065
5066 // Long Immediate zero
5067 operand immL0()
5068 %{
5069 predicate(n->get_long() == 0L);
5070 match(ConL);
5071
5072 op_cost(10);
5073 format %{ %}
5074 interface(CONST_INTER);
5075 %}
5076
5077 // Constant for increment
5078 operand immL1()
5079 %{
5080 predicate(n->get_long() == 1);
5081 match(ConL);
5082
5083 format %{ %}
5084 interface(CONST_INTER);
5085 %}
5086
5087 // Constant for decrement
5088 operand immL_M1()
5089 %{
5090 predicate(n->get_long() == -1);
5091 match(ConL);
5092
5093 format %{ %}
5094 interface(CONST_INTER);
5095 %}
5096
5097 // Long Immediate: low 32-bit mask
5098 operand immL_32bits()
5099 %{
5100 predicate(n->get_long() == 0xFFFFFFFFL);
5101 match(ConL);
5102 op_cost(20);
5103
5104 format %{ %}
5105 interface(CONST_INTER);
5106 %}
5107
5108 // Int Immediate: 2^n-1, positive
5109 operand immI_Pow2M1()
5110 %{
5111 predicate((n->get_int() > 0)
5112 && is_power_of_2((juint)n->get_int() + 1));
5113 match(ConI);
5114
5115 op_cost(20);
5116 format %{ %}
5117 interface(CONST_INTER);
5118 %}
5119
5120 // Float Immediate zero
5121 operand immF0()
5122 %{
5123 predicate(jint_cast(n->getf()) == 0);
5124 match(ConF);
5125
5126 op_cost(5);
5127 format %{ %}
5128 interface(CONST_INTER);
5129 %}
5130
5131 // Float Immediate
5132 operand immF()
5133 %{
5134 match(ConF);
5135
5136 op_cost(15);
5137 format %{ %}
5138 interface(CONST_INTER);
5139 %}
5140
5141 // Half Float Immediate
5142 operand immH()
5143 %{
5144 match(ConH);
5145
5146 op_cost(15);
5147 format %{ %}
5148 interface(CONST_INTER);
5149 %}
5150
5151 // Double Immediate zero
5152 operand immD0()
5153 %{
5154 predicate(jlong_cast(n->getd()) == 0);
5155 match(ConD);
5156
5157 op_cost(5);
5158 format %{ %}
5159 interface(CONST_INTER);
5160 %}
5161
5162 // Double Immediate
5163 operand immD()
5164 %{
5165 match(ConD);
5166
5167 op_cost(15);
5168 format %{ %}
5169 interface(CONST_INTER);
5170 %}
5171
5172 // Immediates for special shifts (sign extend)
5173
5174 // Constants for increment
5175 operand immI_16()
5176 %{
5177 predicate(n->get_int() == 16);
5178 match(ConI);
5179
5180 format %{ %}
5181 interface(CONST_INTER);
5182 %}
5183
5184 operand immI_24()
5185 %{
5186 predicate(n->get_int() == 24);
5187 match(ConI);
5188
5189 format %{ %}
5190 interface(CONST_INTER);
5191 %}
5192
5193 // Constant for byte-wide masking
5194 operand immI_255()
5195 %{
5196 predicate(n->get_int() == 255);
5197 match(ConI);
5198
5199 format %{ %}
5200 interface(CONST_INTER);
5201 %}
5202
5203 // Constant for short-wide masking
5204 operand immI_65535()
5205 %{
5206 predicate(n->get_int() == 65535);
5207 match(ConI);
5208
5209 format %{ %}
5210 interface(CONST_INTER);
5211 %}
5212
5213 // Constant for byte-wide masking
5214 operand immL_255()
5215 %{
5216 predicate(n->get_long() == 255);
5217 match(ConL);
5218
5219 format %{ %}
5220 interface(CONST_INTER);
5221 %}
5222
5223 // Constant for short-wide masking
5224 operand immL_65535()
5225 %{
5226 predicate(n->get_long() == 65535);
5227 match(ConL);
5228
5229 format %{ %}
5230 interface(CONST_INTER);
5231 %}
5232
5233 // AOT Runtime Constants Address
5234 operand immAOTRuntimeConstantsAddress()
5235 %{
5236 // Check if the address is in the range of AOT Runtime Constants
5237 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
5238 match(ConP);
5239
5240 op_cost(0);
5241 format %{ %}
5242 interface(CONST_INTER);
5243 %}
5244
5245 operand kReg()
5246 %{
5247 constraint(ALLOC_IN_RC(vectmask_reg));
5248 match(RegVectMask);
5249 format %{%}
5250 interface(REG_INTER);
5251 %}
5252
5253 // Register Operands
5254 // Integer Register
5255 operand rRegI()
5256 %{
5257 constraint(ALLOC_IN_RC(int_reg));
5258 match(RegI);
5259
5260 match(rax_RegI);
5261 match(rbx_RegI);
5262 match(rcx_RegI);
5263 match(rdx_RegI);
5264 match(rdi_RegI);
5265
5266 format %{ %}
5267 interface(REG_INTER);
5268 %}
5269
5270 // Special Registers
5271 operand rax_RegI()
5272 %{
5273 constraint(ALLOC_IN_RC(int_rax_reg));
5274 match(RegI);
5275 match(rRegI);
5276
5277 format %{ "RAX" %}
5278 interface(REG_INTER);
5279 %}
5280
5281 // Special Registers
5282 operand rbx_RegI()
5283 %{
5284 constraint(ALLOC_IN_RC(int_rbx_reg));
5285 match(RegI);
5286 match(rRegI);
5287
5288 format %{ "RBX" %}
5289 interface(REG_INTER);
5290 %}
5291
5292 operand rcx_RegI()
5293 %{
5294 constraint(ALLOC_IN_RC(int_rcx_reg));
5295 match(RegI);
5296 match(rRegI);
5297
5298 format %{ "RCX" %}
5299 interface(REG_INTER);
5300 %}
5301
5302 operand rdx_RegI()
5303 %{
5304 constraint(ALLOC_IN_RC(int_rdx_reg));
5305 match(RegI);
5306 match(rRegI);
5307
5308 format %{ "RDX" %}
5309 interface(REG_INTER);
5310 %}
5311
5312 operand rdi_RegI()
5313 %{
5314 constraint(ALLOC_IN_RC(int_rdi_reg));
5315 match(RegI);
5316 match(rRegI);
5317
5318 format %{ "RDI" %}
5319 interface(REG_INTER);
5320 %}
5321
5322 operand no_rax_rdx_RegI()
5323 %{
5324 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
5325 match(RegI);
5326 match(rbx_RegI);
5327 match(rcx_RegI);
5328 match(rdi_RegI);
5329
5330 format %{ %}
5331 interface(REG_INTER);
5332 %}
5333
5334 operand no_rbp_r13_RegI()
5335 %{
5336 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
5337 match(RegI);
5338 match(rRegI);
5339 match(rax_RegI);
5340 match(rbx_RegI);
5341 match(rcx_RegI);
5342 match(rdx_RegI);
5343 match(rdi_RegI);
5344
5345 format %{ %}
5346 interface(REG_INTER);
5347 %}
5348
5349 // Pointer Register
5350 operand any_RegP()
5351 %{
5352 constraint(ALLOC_IN_RC(any_reg));
5353 match(RegP);
5354 match(rax_RegP);
5355 match(rbx_RegP);
5356 match(rdi_RegP);
5357 match(rsi_RegP);
5358 match(rbp_RegP);
5359 match(r15_RegP);
5360 match(rRegP);
5361
5362 format %{ %}
5363 interface(REG_INTER);
5364 %}
5365
5366 operand rRegP()
5367 %{
5368 constraint(ALLOC_IN_RC(ptr_reg));
5369 match(RegP);
5370 match(rax_RegP);
5371 match(rbx_RegP);
5372 match(rdi_RegP);
5373 match(rsi_RegP);
5374 match(rbp_RegP); // See Q&A below about
5375 match(r15_RegP); // r15_RegP and rbp_RegP.
5376
5377 format %{ %}
5378 interface(REG_INTER);
5379 %}
5380
5381 operand rRegN() %{
5382 constraint(ALLOC_IN_RC(int_reg));
5383 match(RegN);
5384
5385 format %{ %}
5386 interface(REG_INTER);
5387 %}
5388
5389 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
5390 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
5391 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
5392 // The output of an instruction is controlled by the allocator, which respects
5393 // register class masks, not match rules. Unless an instruction mentions
5394 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
5395 // by the allocator as an input.
5396 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
5397 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
5398 // result, RBP is not included in the output of the instruction either.
5399
5400 // This operand is not allowed to use RBP even if
5401 // RBP is not used to hold the frame pointer.
5402 operand no_rbp_RegP()
5403 %{
5404 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
5405 match(RegP);
5406 match(rbx_RegP);
5407 match(rsi_RegP);
5408 match(rdi_RegP);
5409
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 // Special Registers
5415 // Return a pointer value
5416 operand rax_RegP()
5417 %{
5418 constraint(ALLOC_IN_RC(ptr_rax_reg));
5419 match(RegP);
5420 match(rRegP);
5421
5422 format %{ %}
5423 interface(REG_INTER);
5424 %}
5425
5426 // Special Registers
5427 // Return a compressed pointer value
5428 operand rax_RegN()
5429 %{
5430 constraint(ALLOC_IN_RC(int_rax_reg));
5431 match(RegN);
5432 match(rRegN);
5433
5434 format %{ %}
5435 interface(REG_INTER);
5436 %}
5437
5438 // Used in AtomicAdd
5439 operand rbx_RegP()
5440 %{
5441 constraint(ALLOC_IN_RC(ptr_rbx_reg));
5442 match(RegP);
5443 match(rRegP);
5444
5445 format %{ %}
5446 interface(REG_INTER);
5447 %}
5448
5449 operand rsi_RegP()
5450 %{
5451 constraint(ALLOC_IN_RC(ptr_rsi_reg));
5452 match(RegP);
5453 match(rRegP);
5454
5455 format %{ %}
5456 interface(REG_INTER);
5457 %}
5458
5459 operand rbp_RegP()
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_rbp_reg));
5462 match(RegP);
5463 match(rRegP);
5464
5465 format %{ %}
5466 interface(REG_INTER);
5467 %}
5468
5469 // Used in rep stosq
5470 operand rdi_RegP()
5471 %{
5472 constraint(ALLOC_IN_RC(ptr_rdi_reg));
5473 match(RegP);
5474 match(rRegP);
5475
5476 format %{ %}
5477 interface(REG_INTER);
5478 %}
5479
5480 operand r15_RegP()
5481 %{
5482 constraint(ALLOC_IN_RC(ptr_r15_reg));
5483 match(RegP);
5484 match(rRegP);
5485
5486 format %{ %}
5487 interface(REG_INTER);
5488 %}
5489
5490 operand rRegL()
5491 %{
5492 constraint(ALLOC_IN_RC(long_reg));
5493 match(RegL);
5494 match(rax_RegL);
5495 match(rdx_RegL);
5496
5497 format %{ %}
5498 interface(REG_INTER);
5499 %}
5500
5501 // Special Registers
5502 operand no_rax_rdx_RegL()
5503 %{
5504 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
5505 match(RegL);
5506 match(rRegL);
5507
5508 format %{ %}
5509 interface(REG_INTER);
5510 %}
5511
5512 operand rax_RegL()
5513 %{
5514 constraint(ALLOC_IN_RC(long_rax_reg));
5515 match(RegL);
5516 match(rRegL);
5517
5518 format %{ "RAX" %}
5519 interface(REG_INTER);
5520 %}
5521
5522 operand rcx_RegL()
5523 %{
5524 constraint(ALLOC_IN_RC(long_rcx_reg));
5525 match(RegL);
5526 match(rRegL);
5527
5528 format %{ %}
5529 interface(REG_INTER);
5530 %}
5531
5532 operand rdx_RegL()
5533 %{
5534 constraint(ALLOC_IN_RC(long_rdx_reg));
5535 match(RegL);
5536 match(rRegL);
5537
5538 format %{ %}
5539 interface(REG_INTER);
5540 %}
5541
5542 operand r11_RegL()
5543 %{
5544 constraint(ALLOC_IN_RC(long_r11_reg));
5545 match(RegL);
5546 match(rRegL);
5547
5548 format %{ %}
5549 interface(REG_INTER);
5550 %}
5551
5552 operand no_rbp_r13_RegL()
5553 %{
5554 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
5555 match(RegL);
5556 match(rRegL);
5557 match(rax_RegL);
5558 match(rcx_RegL);
5559 match(rdx_RegL);
5560
5561 format %{ %}
5562 interface(REG_INTER);
5563 %}
5564
5565 // Flags register, used as output of compare instructions
5566 operand rFlagsReg()
5567 %{
5568 constraint(ALLOC_IN_RC(int_flags));
5569 match(RegFlags);
5570
5571 format %{ "RFLAGS" %}
5572 interface(REG_INTER);
5573 %}
5574
5575 // Flags register, used as output of FLOATING POINT compare instructions
5576 operand rFlagsRegU()
5577 %{
5578 constraint(ALLOC_IN_RC(int_flags));
5579 match(RegFlags);
5580
5581 format %{ "RFLAGS_U" %}
5582 interface(REG_INTER);
5583 %}
5584
5585 operand rFlagsRegUCF() %{
5586 constraint(ALLOC_IN_RC(int_flags));
5587 match(RegFlags);
5588 predicate(!UseAPX || !VM_Version::supports_avx10_2());
5589
5590 format %{ "RFLAGS_U_CF" %}
5591 interface(REG_INTER);
5592 %}
5593
5594 operand rFlagsRegUCFE() %{
5595 constraint(ALLOC_IN_RC(int_flags));
5596 match(RegFlags);
5597 predicate(UseAPX && VM_Version::supports_avx10_2());
5598
5599 format %{ "RFLAGS_U_CFE" %}
5600 interface(REG_INTER);
5601 %}
5602
5603 // Float register operands
5604 operand regF() %{
5605 constraint(ALLOC_IN_RC(float_reg));
5606 match(RegF);
5607
5608 format %{ %}
5609 interface(REG_INTER);
5610 %}
5611
5612 // Float register operands
5613 operand legRegF() %{
5614 constraint(ALLOC_IN_RC(float_reg_legacy));
5615 match(RegF);
5616
5617 format %{ %}
5618 interface(REG_INTER);
5619 %}
5620
5621 // Float register operands
5622 operand vlRegF() %{
5623 constraint(ALLOC_IN_RC(float_reg_vl));
5624 match(RegF);
5625
5626 format %{ %}
5627 interface(REG_INTER);
5628 %}
5629
5630 // Double register operands
5631 operand regD() %{
5632 constraint(ALLOC_IN_RC(double_reg));
5633 match(RegD);
5634
5635 format %{ %}
5636 interface(REG_INTER);
5637 %}
5638
5639 // Double register operands
5640 operand legRegD() %{
5641 constraint(ALLOC_IN_RC(double_reg_legacy));
5642 match(RegD);
5643
5644 format %{ %}
5645 interface(REG_INTER);
5646 %}
5647
5648 // Double register operands
5649 operand vlRegD() %{
5650 constraint(ALLOC_IN_RC(double_reg_vl));
5651 match(RegD);
5652
5653 format %{ %}
5654 interface(REG_INTER);
5655 %}
5656
5657 //----------Memory Operands----------------------------------------------------
5658 // Direct Memory Operand
5659 // operand direct(immP addr)
5660 // %{
5661 // match(addr);
5662
5663 // format %{ "[$addr]" %}
5664 // interface(MEMORY_INTER) %{
5665 // base(0xFFFFFFFF);
5666 // index(0x4);
5667 // scale(0x0);
5668 // disp($addr);
5669 // %}
5670 // %}
5671
5672 // Indirect Memory Operand
5673 operand indirect(any_RegP reg)
5674 %{
5675 constraint(ALLOC_IN_RC(ptr_reg));
5676 match(reg);
5677
5678 format %{ "[$reg]" %}
5679 interface(MEMORY_INTER) %{
5680 base($reg);
5681 index(0x4);
5682 scale(0x0);
5683 disp(0x0);
5684 %}
5685 %}
5686
5687 // Indirect Memory Plus Short Offset Operand
5688 operand indOffset8(any_RegP reg, immL8 off)
5689 %{
5690 constraint(ALLOC_IN_RC(ptr_reg));
5691 match(AddP reg off);
5692
5693 format %{ "[$reg + $off (8-bit)]" %}
5694 interface(MEMORY_INTER) %{
5695 base($reg);
5696 index(0x4);
5697 scale(0x0);
5698 disp($off);
5699 %}
5700 %}
5701
5702 // Indirect Memory Plus Long Offset Operand
5703 operand indOffset32(any_RegP reg, immL32 off)
5704 %{
5705 constraint(ALLOC_IN_RC(ptr_reg));
5706 match(AddP reg off);
5707
5708 format %{ "[$reg + $off (32-bit)]" %}
5709 interface(MEMORY_INTER) %{
5710 base($reg);
5711 index(0x4);
5712 scale(0x0);
5713 disp($off);
5714 %}
5715 %}
5716
5717 // Indirect Memory Plus Index Register Plus Offset Operand
5718 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
5719 %{
5720 constraint(ALLOC_IN_RC(ptr_reg));
5721 match(AddP (AddP reg lreg) off);
5722
5723 op_cost(10);
5724 format %{"[$reg + $off + $lreg]" %}
5725 interface(MEMORY_INTER) %{
5726 base($reg);
5727 index($lreg);
5728 scale(0x0);
5729 disp($off);
5730 %}
5731 %}
5732
5733 // Indirect Memory Plus Index Register Plus Offset Operand
5734 operand indIndex(any_RegP reg, rRegL lreg)
5735 %{
5736 constraint(ALLOC_IN_RC(ptr_reg));
5737 match(AddP reg lreg);
5738
5739 op_cost(10);
5740 format %{"[$reg + $lreg]" %}
5741 interface(MEMORY_INTER) %{
5742 base($reg);
5743 index($lreg);
5744 scale(0x0);
5745 disp(0x0);
5746 %}
5747 %}
5748
5749 // Indirect Memory Times Scale Plus Index Register
5750 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
5751 %{
5752 constraint(ALLOC_IN_RC(ptr_reg));
5753 match(AddP reg (LShiftL lreg scale));
5754
5755 op_cost(10);
5756 format %{"[$reg + $lreg << $scale]" %}
5757 interface(MEMORY_INTER) %{
5758 base($reg);
5759 index($lreg);
5760 scale($scale);
5761 disp(0x0);
5762 %}
5763 %}
5764
5765 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
5766 %{
5767 constraint(ALLOC_IN_RC(ptr_reg));
5768 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5769 match(AddP reg (LShiftL (ConvI2L idx) scale));
5770
5771 op_cost(10);
5772 format %{"[$reg + pos $idx << $scale]" %}
5773 interface(MEMORY_INTER) %{
5774 base($reg);
5775 index($idx);
5776 scale($scale);
5777 disp(0x0);
5778 %}
5779 %}
5780
5781 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5782 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
5783 %{
5784 constraint(ALLOC_IN_RC(ptr_reg));
5785 match(AddP (AddP reg (LShiftL lreg scale)) off);
5786
5787 op_cost(10);
5788 format %{"[$reg + $off + $lreg << $scale]" %}
5789 interface(MEMORY_INTER) %{
5790 base($reg);
5791 index($lreg);
5792 scale($scale);
5793 disp($off);
5794 %}
5795 %}
5796
5797 // Indirect Memory Plus Positive Index Register Plus Offset Operand
5798 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
5799 %{
5800 constraint(ALLOC_IN_RC(ptr_reg));
5801 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5802 match(AddP (AddP reg (ConvI2L idx)) off);
5803
5804 op_cost(10);
5805 format %{"[$reg + $off + $idx]" %}
5806 interface(MEMORY_INTER) %{
5807 base($reg);
5808 index($idx);
5809 scale(0x0);
5810 disp($off);
5811 %}
5812 %}
5813
5814 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5815 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5816 %{
5817 constraint(ALLOC_IN_RC(ptr_reg));
5818 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5819 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5820
5821 op_cost(10);
5822 format %{"[$reg + $off + $idx << $scale]" %}
5823 interface(MEMORY_INTER) %{
5824 base($reg);
5825 index($idx);
5826 scale($scale);
5827 disp($off);
5828 %}
5829 %}
5830
5831 // Indirect Narrow Oop Operand
5832 operand indCompressedOop(rRegN reg) %{
5833 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5834 constraint(ALLOC_IN_RC(ptr_reg));
5835 match(DecodeN reg);
5836
5837 op_cost(10);
5838 format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
5839 interface(MEMORY_INTER) %{
5840 base(0xc); // R12
5841 index($reg);
5842 scale(0x3);
5843 disp(0x0);
5844 %}
5845 %}
5846
5847 // Indirect Narrow Oop Plus Offset Operand
5848 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5849 // we can't free r12 even with CompressedOops::base() == nullptr.
5850 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5851 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5852 constraint(ALLOC_IN_RC(ptr_reg));
5853 match(AddP (DecodeN reg) off);
5854
5855 op_cost(10);
5856 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5857 interface(MEMORY_INTER) %{
5858 base(0xc); // R12
5859 index($reg);
5860 scale(0x3);
5861 disp($off);
5862 %}
5863 %}
5864
5865 // Indirect Memory Operand
5866 operand indirectNarrow(rRegN reg)
5867 %{
5868 predicate(CompressedOops::shift() == 0);
5869 constraint(ALLOC_IN_RC(ptr_reg));
5870 match(DecodeN reg);
5871
5872 format %{ "[$reg]" %}
5873 interface(MEMORY_INTER) %{
5874 base($reg);
5875 index(0x4);
5876 scale(0x0);
5877 disp(0x0);
5878 %}
5879 %}
5880
5881 // Indirect Memory Plus Short Offset Operand
5882 operand indOffset8Narrow(rRegN reg, immL8 off)
5883 %{
5884 predicate(CompressedOops::shift() == 0);
5885 constraint(ALLOC_IN_RC(ptr_reg));
5886 match(AddP (DecodeN reg) off);
5887
5888 format %{ "[$reg + $off (8-bit)]" %}
5889 interface(MEMORY_INTER) %{
5890 base($reg);
5891 index(0x4);
5892 scale(0x0);
5893 disp($off);
5894 %}
5895 %}
5896
5897 // Indirect Memory Plus Long Offset Operand
5898 operand indOffset32Narrow(rRegN reg, immL32 off)
5899 %{
5900 predicate(CompressedOops::shift() == 0);
5901 constraint(ALLOC_IN_RC(ptr_reg));
5902 match(AddP (DecodeN reg) off);
5903
5904 format %{ "[$reg + $off (32-bit)]" %}
5905 interface(MEMORY_INTER) %{
5906 base($reg);
5907 index(0x4);
5908 scale(0x0);
5909 disp($off);
5910 %}
5911 %}
5912
5913 // Indirect Memory Plus Index Register Plus Offset Operand
5914 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
5915 %{
5916 predicate(CompressedOops::shift() == 0);
5917 constraint(ALLOC_IN_RC(ptr_reg));
5918 match(AddP (AddP (DecodeN reg) lreg) off);
5919
5920 op_cost(10);
5921 format %{"[$reg + $off + $lreg]" %}
5922 interface(MEMORY_INTER) %{
5923 base($reg);
5924 index($lreg);
5925 scale(0x0);
5926 disp($off);
5927 %}
5928 %}
5929
5930 // Indirect Memory Plus Index Register Plus Offset Operand
5931 operand indIndexNarrow(rRegN reg, rRegL lreg)
5932 %{
5933 predicate(CompressedOops::shift() == 0);
5934 constraint(ALLOC_IN_RC(ptr_reg));
5935 match(AddP (DecodeN reg) lreg);
5936
5937 op_cost(10);
5938 format %{"[$reg + $lreg]" %}
5939 interface(MEMORY_INTER) %{
5940 base($reg);
5941 index($lreg);
5942 scale(0x0);
5943 disp(0x0);
5944 %}
5945 %}
5946
5947 // Indirect Memory Times Scale Plus Index Register
5948 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5949 %{
5950 predicate(CompressedOops::shift() == 0);
5951 constraint(ALLOC_IN_RC(ptr_reg));
5952 match(AddP (DecodeN reg) (LShiftL lreg scale));
5953
5954 op_cost(10);
5955 format %{"[$reg + $lreg << $scale]" %}
5956 interface(MEMORY_INTER) %{
5957 base($reg);
5958 index($lreg);
5959 scale($scale);
5960 disp(0x0);
5961 %}
5962 %}
5963
5964 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5965 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5966 %{
5967 predicate(CompressedOops::shift() == 0);
5968 constraint(ALLOC_IN_RC(ptr_reg));
5969 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5970
5971 op_cost(10);
5972 format %{"[$reg + $off + $lreg << $scale]" %}
5973 interface(MEMORY_INTER) %{
5974 base($reg);
5975 index($lreg);
5976 scale($scale);
5977 disp($off);
5978 %}
5979 %}
5980
5981 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
5982 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
5983 %{
5984 constraint(ALLOC_IN_RC(ptr_reg));
5985 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5986 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
5987
5988 op_cost(10);
5989 format %{"[$reg + $off + $idx]" %}
5990 interface(MEMORY_INTER) %{
5991 base($reg);
5992 index($idx);
5993 scale(0x0);
5994 disp($off);
5995 %}
5996 %}
5997
5998 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5999 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
6000 %{
6001 constraint(ALLOC_IN_RC(ptr_reg));
6002 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
6003 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
6004
6005 op_cost(10);
6006 format %{"[$reg + $off + $idx << $scale]" %}
6007 interface(MEMORY_INTER) %{
6008 base($reg);
6009 index($idx);
6010 scale($scale);
6011 disp($off);
6012 %}
6013 %}
6014
6015 //----------Special Memory Operands--------------------------------------------
6016 // Stack Slot Operand - This operand is used for loading and storing temporary
6017 // values on the stack where a match requires a value to
6018 // flow through memory.
6019 operand stackSlotP(sRegP reg)
6020 %{
6021 constraint(ALLOC_IN_RC(stack_slots));
6022 // No match rule because this operand is only generated in matching
6023
6024 format %{ "[$reg]" %}
6025 interface(MEMORY_INTER) %{
6026 base(0x4); // RSP
6027 index(0x4); // No Index
6028 scale(0x0); // No Scale
6029 disp($reg); // Stack Offset
6030 %}
6031 %}
6032
6033 operand stackSlotI(sRegI reg)
6034 %{
6035 constraint(ALLOC_IN_RC(stack_slots));
6036 // No match rule because this operand is only generated in matching
6037
6038 format %{ "[$reg]" %}
6039 interface(MEMORY_INTER) %{
6040 base(0x4); // RSP
6041 index(0x4); // No Index
6042 scale(0x0); // No Scale
6043 disp($reg); // Stack Offset
6044 %}
6045 %}
6046
6047 operand stackSlotF(sRegF reg)
6048 %{
6049 constraint(ALLOC_IN_RC(stack_slots));
6050 // No match rule because this operand is only generated in matching
6051
6052 format %{ "[$reg]" %}
6053 interface(MEMORY_INTER) %{
6054 base(0x4); // RSP
6055 index(0x4); // No Index
6056 scale(0x0); // No Scale
6057 disp($reg); // Stack Offset
6058 %}
6059 %}
6060
6061 operand stackSlotD(sRegD reg)
6062 %{
6063 constraint(ALLOC_IN_RC(stack_slots));
6064 // No match rule because this operand is only generated in matching
6065
6066 format %{ "[$reg]" %}
6067 interface(MEMORY_INTER) %{
6068 base(0x4); // RSP
6069 index(0x4); // No Index
6070 scale(0x0); // No Scale
6071 disp($reg); // Stack Offset
6072 %}
6073 %}
6074 operand stackSlotL(sRegL reg)
6075 %{
6076 constraint(ALLOC_IN_RC(stack_slots));
6077 // No match rule because this operand is only generated in matching
6078
6079 format %{ "[$reg]" %}
6080 interface(MEMORY_INTER) %{
6081 base(0x4); // RSP
6082 index(0x4); // No Index
6083 scale(0x0); // No Scale
6084 disp($reg); // Stack Offset
6085 %}
6086 %}
6087
6088 //----------Conditional Branch Operands----------------------------------------
6089 // Comparison Op - This is the operation of the comparison, and is limited to
6090 // the following set of codes:
6091 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6092 //
6093 // Other attributes of the comparison, such as unsignedness, are specified
6094 // by the comparison instruction that sets a condition code flags register.
6095 // That result is represented by a flags operand whose subtype is appropriate
6096 // to the unsignedness (etc.) of the comparison.
6097 //
6098 // Later, the instruction which matches both the Comparison Op (a Bool) and
6099 // the flags (produced by the Cmp) specifies the coding of the comparison op
6100 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6101
6102 // Comparison Code
6103 operand cmpOp()
6104 %{
6105 match(Bool);
6106
6107 format %{ "" %}
6108 interface(COND_INTER) %{
6109 equal(0x4, "e");
6110 not_equal(0x5, "ne");
6111 less(0xc, "l");
6112 greater_equal(0xd, "ge");
6113 less_equal(0xe, "le");
6114 greater(0xf, "g");
6115 overflow(0x0, "o");
6116 no_overflow(0x1, "no");
6117 %}
6118 %}
6119
6120 // Comparison Code, unsigned compare. Used by FP also, with
6121 // C2 (unordered) turned into GT or LT already. The other bits
6122 // C0 and C3 are turned into Carry & Zero flags.
6123 operand cmpOpU()
6124 %{
6125 match(Bool);
6126
6127 format %{ "" %}
6128 interface(COND_INTER) %{
6129 equal(0x4, "e");
6130 not_equal(0x5, "ne");
6131 less(0x2, "b");
6132 greater_equal(0x3, "ae");
6133 less_equal(0x6, "be");
6134 greater(0x7, "a");
6135 overflow(0x0, "o");
6136 no_overflow(0x1, "no");
6137 %}
6138 %}
6139
6140
6141 // Floating comparisons that don't require any fixup for the unordered case,
6142 // If both inputs of the comparison are the same, ZF is always set so we
6143 // don't need to use cmpOpUCF2 for eq/ne
6144 operand cmpOpUCF() %{
6145 match(Bool);
6146 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6147 (n->as_Bool()->_test._test == BoolTest::lt ||
6148 n->as_Bool()->_test._test == BoolTest::ge ||
6149 n->as_Bool()->_test._test == BoolTest::le ||
6150 n->as_Bool()->_test._test == BoolTest::gt ||
6151 n->in(1)->in(1) == n->in(1)->in(2)));
6152 format %{ "" %}
6153 interface(COND_INTER) %{
6154 equal(0xb, "np");
6155 not_equal(0xa, "p");
6156 less(0x2, "b");
6157 greater_equal(0x3, "ae");
6158 less_equal(0x6, "be");
6159 greater(0x7, "a");
6160 overflow(0x0, "o");
6161 no_overflow(0x1, "no");
6162 %}
6163 %}
6164
6165
6166 // Floating comparisons that can be fixed up with extra conditional jumps
6167 operand cmpOpUCF2() %{
6168 match(Bool);
6169 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6170 (n->as_Bool()->_test._test == BoolTest::ne ||
6171 n->as_Bool()->_test._test == BoolTest::eq) &&
6172 n->in(1)->in(1) != n->in(1)->in(2));
6173 format %{ "" %}
6174 interface(COND_INTER) %{
6175 equal(0x4, "e");
6176 not_equal(0x5, "ne");
6177 less(0x2, "b");
6178 greater_equal(0x3, "ae");
6179 less_equal(0x6, "be");
6180 greater(0x7, "a");
6181 overflow(0x0, "o");
6182 no_overflow(0x1, "no");
6183 %}
6184 %}
6185
6186
6187 // Floating point comparisons that set condition flags to test more directly,
6188 // Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
6189 // are used for L (<) and LE (<=) conditions. It's important to convert these
6190 // latter conditions to ones that use unsigned tests before passing into an
6191 // instruction because the preceding comparison might be based on a three way
6192 // comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
6193 operand cmpOpUCFE()
6194 %{
6195 match(Bool);
6196 predicate((UseAPX && VM_Version::supports_avx10_2()) &&
6197 (n->as_Bool()->_test._test == BoolTest::ne ||
6198 n->as_Bool()->_test._test == BoolTest::eq ||
6199 n->as_Bool()->_test._test == BoolTest::lt ||
6200 n->as_Bool()->_test._test == BoolTest::ge ||
6201 n->as_Bool()->_test._test == BoolTest::le ||
6202 n->as_Bool()->_test._test == BoolTest::gt));
6203
6204 format %{ "" %}
6205 interface(COND_INTER) %{
6206 equal(0x4, "e");
6207 not_equal(0x5, "ne");
6208 less(0x2, "b");
6209 greater_equal(0x3, "ae");
6210 less_equal(0x6, "be");
6211 greater(0x7, "a");
6212 overflow(0x0, "o");
6213 no_overflow(0x1, "no");
6214 %}
6215 %}
6216
6217 // Operands for bound floating pointer register arguments
6218 operand rxmm0() %{
6219 constraint(ALLOC_IN_RC(xmm0_reg));
6220 match(VecX);
6221 format%{%}
6222 interface(REG_INTER);
6223 %}
6224
6225 // Vectors
6226
6227 // Dummy generic vector class. Should be used for all vector operands.
6228 // Replaced with vec[SDXYZ] during post-selection pass.
6229 operand vec() %{
6230 constraint(ALLOC_IN_RC(dynamic));
6231 match(VecX);
6232 match(VecY);
6233 match(VecZ);
6234 match(VecS);
6235 match(VecD);
6236
6237 format %{ %}
6238 interface(REG_INTER);
6239 %}
6240
6241 // Dummy generic legacy vector class. Should be used for all legacy vector operands.
6242 // Replaced with legVec[SDXYZ] during post-selection cleanup.
6243 // Note: legacy register class is used to avoid extra (unneeded in 32-bit VM)
6244 // runtime code generation via reg_class_dynamic.
6245 operand legVec() %{
6246 constraint(ALLOC_IN_RC(dynamic));
6247 match(VecX);
6248 match(VecY);
6249 match(VecZ);
6250 match(VecS);
6251 match(VecD);
6252
6253 format %{ %}
6254 interface(REG_INTER);
6255 %}
6256
6257 // Replaces vec during post-selection cleanup. See above.
6258 operand vecS() %{
6259 constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
6260 match(VecS);
6261
6262 format %{ %}
6263 interface(REG_INTER);
6264 %}
6265
6266 // Replaces legVec during post-selection cleanup. See above.
6267 operand legVecS() %{
6268 constraint(ALLOC_IN_RC(vectors_reg_legacy));
6269 match(VecS);
6270
6271 format %{ %}
6272 interface(REG_INTER);
6273 %}
6274
6275 // Replaces vec during post-selection cleanup. See above.
6276 operand vecD() %{
6277 constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
6278 match(VecD);
6279
6280 format %{ %}
6281 interface(REG_INTER);
6282 %}
6283
6284 // Replaces legVec during post-selection cleanup. See above.
6285 operand legVecD() %{
6286 constraint(ALLOC_IN_RC(vectord_reg_legacy));
6287 match(VecD);
6288
6289 format %{ %}
6290 interface(REG_INTER);
6291 %}
6292
6293 // Replaces vec during post-selection cleanup. See above.
6294 operand vecX() %{
6295 constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
6296 match(VecX);
6297
6298 format %{ %}
6299 interface(REG_INTER);
6300 %}
6301
6302 // Replaces legVec during post-selection cleanup. See above.
6303 operand legVecX() %{
6304 constraint(ALLOC_IN_RC(vectorx_reg_legacy));
6305 match(VecX);
6306
6307 format %{ %}
6308 interface(REG_INTER);
6309 %}
6310
6311 // Replaces vec during post-selection cleanup. See above.
6312 operand vecY() %{
6313 constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
6314 match(VecY);
6315
6316 format %{ %}
6317 interface(REG_INTER);
6318 %}
6319
6320 // Replaces legVec during post-selection cleanup. See above.
6321 operand legVecY() %{
6322 constraint(ALLOC_IN_RC(vectory_reg_legacy));
6323 match(VecY);
6324
6325 format %{ %}
6326 interface(REG_INTER);
6327 %}
6328
6329 // Replaces vec during post-selection cleanup. See above.
6330 operand vecZ() %{
6331 constraint(ALLOC_IN_RC(vectorz_reg));
6332 match(VecZ);
6333
6334 format %{ %}
6335 interface(REG_INTER);
6336 %}
6337
6338 // Replaces legVec during post-selection cleanup. See above.
6339 operand legVecZ() %{
6340 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6341 match(VecZ);
6342
6343 format %{ %}
6344 interface(REG_INTER);
6345 %}
6346
6347 //----------OPERAND CLASSES----------------------------------------------------
6348 // Operand Classes are groups of operands that are used as to simplify
6349 // instruction definitions by not requiring the AD writer to specify separate
6350 // instructions for every form of operand when the instruction accepts
6351 // multiple operand types with the same basic encoding and format. The classic
6352 // case of this is memory operands.
6353
6354 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6355 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6356 indCompressedOop, indCompressedOopOffset,
6357 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6358 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6359 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6360
6361 //----------PIPELINE-----------------------------------------------------------
6362 // Rules which define the behavior of the target architectures pipeline.
6363 pipeline %{
6364
6365 //----------ATTRIBUTES---------------------------------------------------------
6366 attributes %{
6367 variable_size_instructions; // Fixed size instructions
6368 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6369 instruction_unit_size = 1; // An instruction is 1 bytes long
6370 instruction_fetch_unit_size = 16; // The processor fetches one line
6371 instruction_fetch_units = 1; // of 16 bytes
6372 %}
6373
6374 //----------RESOURCES----------------------------------------------------------
6375 // Resources are the functional units available to the machine
6376
6377 // Generic P2/P3 pipeline
6378 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
6379 // 3 instructions decoded per cycle.
6380 // 2 load/store ops per cycle, 1 branch, 1 FPU,
6381 // 3 ALU op, only ALU0 handles mul instructions.
6382 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
6383 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
6384 BR, FPU,
6385 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
6386
6387 //----------PIPELINE DESCRIPTION-----------------------------------------------
6388 // Pipeline Description specifies the stages in the machine's pipeline
6389
6390 // Generic P2/P3 pipeline
6391 pipe_desc(S0, S1, S2, S3, S4, S5);
6392
6393 //----------PIPELINE CLASSES---------------------------------------------------
6394 // Pipeline Classes describe the stages in which input and output are
6395 // referenced by the hardware pipeline.
6396
6397 // Naming convention: ialu or fpu
6398 // Then: _reg
6399 // Then: _reg if there is a 2nd register
6400 // Then: _long if it's a pair of instructions implementing a long
6401 // Then: _fat if it requires the big decoder
6402 // Or: _mem if it requires the big decoder and a memory unit.
6403
6404 // Integer ALU reg operation
6405 pipe_class ialu_reg(rRegI dst)
6406 %{
6407 single_instruction;
6408 dst : S4(write);
6409 dst : S3(read);
6410 DECODE : S0; // any decoder
6411 ALU : S3; // any alu
6412 %}
6413
6414 // Long ALU reg operation
6415 pipe_class ialu_reg_long(rRegL dst)
6416 %{
6417 instruction_count(2);
6418 dst : S4(write);
6419 dst : S3(read);
6420 DECODE : S0(2); // any 2 decoders
6421 ALU : S3(2); // both alus
6422 %}
6423
6424 // Integer ALU reg operation using big decoder
6425 pipe_class ialu_reg_fat(rRegI dst)
6426 %{
6427 single_instruction;
6428 dst : S4(write);
6429 dst : S3(read);
6430 D0 : S0; // big decoder only
6431 ALU : S3; // any alu
6432 %}
6433
6434 // Integer ALU reg-reg operation
6435 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
6436 %{
6437 single_instruction;
6438 dst : S4(write);
6439 src : S3(read);
6440 DECODE : S0; // any decoder
6441 ALU : S3; // any alu
6442 %}
6443
6444 // Integer ALU reg-reg operation
6445 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
6446 %{
6447 single_instruction;
6448 dst : S4(write);
6449 src : S3(read);
6450 D0 : S0; // big decoder only
6451 ALU : S3; // any alu
6452 %}
6453
6454 // Integer ALU reg-mem operation
6455 pipe_class ialu_reg_mem(rRegI dst, memory mem)
6456 %{
6457 single_instruction;
6458 dst : S5(write);
6459 mem : S3(read);
6460 D0 : S0; // big decoder only
6461 ALU : S4; // any alu
6462 MEM : S3; // any mem
6463 %}
6464
6465 // Integer mem operation (prefetch)
6466 pipe_class ialu_mem(memory mem)
6467 %{
6468 single_instruction;
6469 mem : S3(read);
6470 D0 : S0; // big decoder only
6471 MEM : S3; // any mem
6472 %}
6473
6474 // Integer Store to Memory
6475 pipe_class ialu_mem_reg(memory mem, rRegI src)
6476 %{
6477 single_instruction;
6478 mem : S3(read);
6479 src : S5(read);
6480 D0 : S0; // big decoder only
6481 ALU : S4; // any alu
6482 MEM : S3;
6483 %}
6484
6485 // // Long Store to Memory
6486 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
6487 // %{
6488 // instruction_count(2);
6489 // mem : S3(read);
6490 // src : S5(read);
6491 // D0 : S0(2); // big decoder only; twice
6492 // ALU : S4(2); // any 2 alus
6493 // MEM : S3(2); // Both mems
6494 // %}
6495
6496 // Integer Store to Memory
6497 pipe_class ialu_mem_imm(memory mem)
6498 %{
6499 single_instruction;
6500 mem : S3(read);
6501 D0 : S0; // big decoder only
6502 ALU : S4; // any alu
6503 MEM : S3;
6504 %}
6505
6506 // Integer ALU0 reg-reg operation
6507 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
6508 %{
6509 single_instruction;
6510 dst : S4(write);
6511 src : S3(read);
6512 D0 : S0; // Big decoder only
6513 ALU0 : S3; // only alu0
6514 %}
6515
6516 // Integer ALU0 reg-mem operation
6517 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
6518 %{
6519 single_instruction;
6520 dst : S5(write);
6521 mem : S3(read);
6522 D0 : S0; // big decoder only
6523 ALU0 : S4; // ALU0 only
6524 MEM : S3; // any mem
6525 %}
6526
6527 // Integer ALU reg-reg operation
6528 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
6529 %{
6530 single_instruction;
6531 cr : S4(write);
6532 src1 : S3(read);
6533 src2 : S3(read);
6534 DECODE : S0; // any decoder
6535 ALU : S3; // any alu
6536 %}
6537
6538 // Integer ALU reg-imm operation
6539 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
6540 %{
6541 single_instruction;
6542 cr : S4(write);
6543 src1 : S3(read);
6544 DECODE : S0; // any decoder
6545 ALU : S3; // any alu
6546 %}
6547
6548 // Integer ALU reg-mem operation
6549 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
6550 %{
6551 single_instruction;
6552 cr : S4(write);
6553 src1 : S3(read);
6554 src2 : S3(read);
6555 D0 : S0; // big decoder only
6556 ALU : S4; // any alu
6557 MEM : S3;
6558 %}
6559
6560 // Conditional move reg-reg
6561 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
6562 %{
6563 instruction_count(4);
6564 y : S4(read);
6565 q : S3(read);
6566 p : S3(read);
6567 DECODE : S0(4); // any decoder
6568 %}
6569
6570 // Conditional move reg-reg
6571 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
6572 %{
6573 single_instruction;
6574 dst : S4(write);
6575 src : S3(read);
6576 cr : S3(read);
6577 DECODE : S0; // any decoder
6578 %}
6579
6580 // Conditional move reg-mem
6581 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
6582 %{
6583 single_instruction;
6584 dst : S4(write);
6585 src : S3(read);
6586 cr : S3(read);
6587 DECODE : S0; // any decoder
6588 MEM : S3;
6589 %}
6590
6591 // Conditional move reg-reg long
6592 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
6593 %{
6594 single_instruction;
6595 dst : S4(write);
6596 src : S3(read);
6597 cr : S3(read);
6598 DECODE : S0(2); // any 2 decoders
6599 %}
6600
6601 // Float reg-reg operation
6602 pipe_class fpu_reg(regD dst)
6603 %{
6604 instruction_count(2);
6605 dst : S3(read);
6606 DECODE : S0(2); // any 2 decoders
6607 FPU : S3;
6608 %}
6609
6610 // Float reg-reg operation
6611 pipe_class fpu_reg_reg(regD dst, regD src)
6612 %{
6613 instruction_count(2);
6614 dst : S4(write);
6615 src : S3(read);
6616 DECODE : S0(2); // any 2 decoders
6617 FPU : S3;
6618 %}
6619
6620 // Float reg-reg operation
6621 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
6622 %{
6623 instruction_count(3);
6624 dst : S4(write);
6625 src1 : S3(read);
6626 src2 : S3(read);
6627 DECODE : S0(3); // any 3 decoders
6628 FPU : S3(2);
6629 %}
6630
6631 // Float reg-reg operation
6632 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
6633 %{
6634 instruction_count(4);
6635 dst : S4(write);
6636 src1 : S3(read);
6637 src2 : S3(read);
6638 src3 : S3(read);
6639 DECODE : S0(4); // any 3 decoders
6640 FPU : S3(2);
6641 %}
6642
6643 // Float reg-reg operation
6644 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
6645 %{
6646 instruction_count(4);
6647 dst : S4(write);
6648 src1 : S3(read);
6649 src2 : S3(read);
6650 src3 : S3(read);
6651 DECODE : S1(3); // any 3 decoders
6652 D0 : S0; // Big decoder only
6653 FPU : S3(2);
6654 MEM : S3;
6655 %}
6656
6657 // Float reg-mem operation
6658 pipe_class fpu_reg_mem(regD dst, memory mem)
6659 %{
6660 instruction_count(2);
6661 dst : S5(write);
6662 mem : S3(read);
6663 D0 : S0; // big decoder only
6664 DECODE : S1; // any decoder for FPU POP
6665 FPU : S4;
6666 MEM : S3; // any mem
6667 %}
6668
6669 // Float reg-mem operation
6670 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
6671 %{
6672 instruction_count(3);
6673 dst : S5(write);
6674 src1 : S3(read);
6675 mem : S3(read);
6676 D0 : S0; // big decoder only
6677 DECODE : S1(2); // any decoder for FPU POP
6678 FPU : S4;
6679 MEM : S3; // any mem
6680 %}
6681
6682 // Float mem-reg operation
6683 pipe_class fpu_mem_reg(memory mem, regD src)
6684 %{
6685 instruction_count(2);
6686 src : S5(read);
6687 mem : S3(read);
6688 DECODE : S0; // any decoder for FPU PUSH
6689 D0 : S1; // big decoder only
6690 FPU : S4;
6691 MEM : S3; // any mem
6692 %}
6693
6694 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
6695 %{
6696 instruction_count(3);
6697 src1 : S3(read);
6698 src2 : S3(read);
6699 mem : S3(read);
6700 DECODE : S0(2); // any decoder for FPU PUSH
6701 D0 : S1; // big decoder only
6702 FPU : S4;
6703 MEM : S3; // any mem
6704 %}
6705
6706 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
6707 %{
6708 instruction_count(3);
6709 src1 : S3(read);
6710 src2 : S3(read);
6711 mem : S4(read);
6712 DECODE : S0; // any decoder for FPU PUSH
6713 D0 : S0(2); // big decoder only
6714 FPU : S4;
6715 MEM : S3(2); // any mem
6716 %}
6717
6718 pipe_class fpu_mem_mem(memory dst, memory src1)
6719 %{
6720 instruction_count(2);
6721 src1 : S3(read);
6722 dst : S4(read);
6723 D0 : S0(2); // big decoder only
6724 MEM : S3(2); // any mem
6725 %}
6726
6727 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
6728 %{
6729 instruction_count(3);
6730 src1 : S3(read);
6731 src2 : S3(read);
6732 dst : S4(read);
6733 D0 : S0(3); // big decoder only
6734 FPU : S4;
6735 MEM : S3(3); // any mem
6736 %}
6737
6738 pipe_class fpu_mem_reg_con(memory mem, regD src1)
6739 %{
6740 instruction_count(3);
6741 src1 : S4(read);
6742 mem : S4(read);
6743 DECODE : S0; // any decoder for FPU PUSH
6744 D0 : S0(2); // big decoder only
6745 FPU : S4;
6746 MEM : S3(2); // any mem
6747 %}
6748
6749 // Float load constant
6750 pipe_class fpu_reg_con(regD dst)
6751 %{
6752 instruction_count(2);
6753 dst : S5(write);
6754 D0 : S0; // big decoder only for the load
6755 DECODE : S1; // any decoder for FPU POP
6756 FPU : S4;
6757 MEM : S3; // any mem
6758 %}
6759
6760 // Float load constant
6761 pipe_class fpu_reg_reg_con(regD dst, regD src)
6762 %{
6763 instruction_count(3);
6764 dst : S5(write);
6765 src : S3(read);
6766 D0 : S0; // big decoder only for the load
6767 DECODE : S1(2); // any decoder for FPU POP
6768 FPU : S4;
6769 MEM : S3; // any mem
6770 %}
6771
6772 // UnConditional branch
6773 pipe_class pipe_jmp(label labl)
6774 %{
6775 single_instruction;
6776 BR : S3;
6777 %}
6778
6779 // Conditional branch
6780 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
6781 %{
6782 single_instruction;
6783 cr : S1(read);
6784 BR : S3;
6785 %}
6786
6787 // Allocation idiom
6788 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
6789 %{
6790 instruction_count(1); force_serialization;
6791 fixed_latency(6);
6792 heap_ptr : S3(read);
6793 DECODE : S0(3);
6794 D0 : S2;
6795 MEM : S3;
6796 ALU : S3(2);
6797 dst : S5(write);
6798 BR : S5;
6799 %}
6800
6801 // Generic big/slow expanded idiom
6802 pipe_class pipe_slow()
6803 %{
6804 instruction_count(10); multiple_bundles; force_serialization;
6805 fixed_latency(100);
6806 D0 : S0(2);
6807 MEM : S3(2);
6808 %}
6809
6810 // The real do-nothing guy
6811 pipe_class empty()
6812 %{
6813 instruction_count(0);
6814 %}
6815
6816 // Define the class for the Nop node
6817 define
6818 %{
6819 MachNop = empty;
6820 %}
6821
6822 %}
6823
6824 //----------INSTRUCTIONS-------------------------------------------------------
6825 //
6826 // match -- States which machine-independent subtree may be replaced
6827 // by this instruction.
6828 // ins_cost -- The estimated cost of this instruction is used by instruction
6829 // selection to identify a minimum cost tree of machine
6830 // instructions that matches a tree of machine-independent
6831 // instructions.
6832 // format -- A string providing the disassembly for this instruction.
6833 // The value of an instruction's operand may be inserted
6834 // by referring to it with a '$' prefix.
6835 // opcode -- Three instruction opcodes may be provided. These are referred
6836 // to within an encode class as $primary, $secondary, and $tertiary
6837 // rrspectively. The primary opcode is commonly used to
6838 // indicate the type of machine instruction, while secondary
6839 // and tertiary are often used for prefix options or addressing
6840 // modes.
6841 // ins_encode -- A list of encode classes with parameters. The encode class
6842 // name must have been defined in an 'enc_class' specification
6843 // in the encode section of the architecture description.
6844
6845 // ============================================================================
6846
6847 instruct ShouldNotReachHere() %{
6848 match(Halt);
6849 format %{ "stop\t# ShouldNotReachHere" %}
6850 ins_encode %{
6851 if (is_reachable()) {
6852 const char* str = __ code_string(_halt_reason);
6853 __ stop(str);
6854 }
6855 %}
6856 ins_pipe(pipe_slow);
6857 %}
6858
6859 // ============================================================================
6860
6861 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
6862 // Load Float
6863 instruct MoveF2VL(vlRegF dst, regF src) %{
6864 match(Set dst src);
6865 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6866 ins_encode %{
6867 ShouldNotReachHere();
6868 %}
6869 ins_pipe( fpu_reg_reg );
6870 %}
6871
6872 // Load Float
6873 instruct MoveF2LEG(legRegF dst, regF src) %{
6874 match(Set dst src);
6875 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6876 ins_encode %{
6877 ShouldNotReachHere();
6878 %}
6879 ins_pipe( fpu_reg_reg );
6880 %}
6881
6882 // Load Float
6883 instruct MoveVL2F(regF dst, vlRegF src) %{
6884 match(Set dst src);
6885 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6886 ins_encode %{
6887 ShouldNotReachHere();
6888 %}
6889 ins_pipe( fpu_reg_reg );
6890 %}
6891
6892 // Load Float
6893 instruct MoveLEG2F(regF dst, legRegF src) %{
6894 match(Set dst src);
6895 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6896 ins_encode %{
6897 ShouldNotReachHere();
6898 %}
6899 ins_pipe( fpu_reg_reg );
6900 %}
6901
6902 // Load Double
6903 instruct MoveD2VL(vlRegD dst, regD src) %{
6904 match(Set dst src);
6905 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6906 ins_encode %{
6907 ShouldNotReachHere();
6908 %}
6909 ins_pipe( fpu_reg_reg );
6910 %}
6911
6912 // Load Double
6913 instruct MoveD2LEG(legRegD dst, regD src) %{
6914 match(Set dst src);
6915 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6916 ins_encode %{
6917 ShouldNotReachHere();
6918 %}
6919 ins_pipe( fpu_reg_reg );
6920 %}
6921
6922 // Load Double
6923 instruct MoveVL2D(regD dst, vlRegD src) %{
6924 match(Set dst src);
6925 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6926 ins_encode %{
6927 ShouldNotReachHere();
6928 %}
6929 ins_pipe( fpu_reg_reg );
6930 %}
6931
6932 // Load Double
6933 instruct MoveLEG2D(regD dst, legRegD src) %{
6934 match(Set dst src);
6935 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6936 ins_encode %{
6937 ShouldNotReachHere();
6938 %}
6939 ins_pipe( fpu_reg_reg );
6940 %}
6941
6942 //----------Load/Store/Move Instructions---------------------------------------
6943 //----------Load Instructions--------------------------------------------------
6944
6945 // Load Byte (8 bit signed)
6946 instruct loadB(rRegI dst, memory mem)
6947 %{
6948 match(Set dst (LoadB mem));
6949
6950 ins_cost(125);
6951 format %{ "movsbl $dst, $mem\t# byte" %}
6952
6953 ins_encode %{
6954 __ movsbl($dst$$Register, $mem$$Address);
6955 %}
6956
6957 ins_pipe(ialu_reg_mem);
6958 %}
6959
6960 // Load Byte (8 bit signed) into Long Register
6961 instruct loadB2L(rRegL dst, memory mem)
6962 %{
6963 match(Set dst (ConvI2L (LoadB mem)));
6964
6965 ins_cost(125);
6966 format %{ "movsbq $dst, $mem\t# byte -> long" %}
6967
6968 ins_encode %{
6969 __ movsbq($dst$$Register, $mem$$Address);
6970 %}
6971
6972 ins_pipe(ialu_reg_mem);
6973 %}
6974
6975 // Load Unsigned Byte (8 bit UNsigned)
6976 instruct loadUB(rRegI dst, memory mem)
6977 %{
6978 match(Set dst (LoadUB mem));
6979
6980 ins_cost(125);
6981 format %{ "movzbl $dst, $mem\t# ubyte" %}
6982
6983 ins_encode %{
6984 __ movzbl($dst$$Register, $mem$$Address);
6985 %}
6986
6987 ins_pipe(ialu_reg_mem);
6988 %}
6989
6990 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6991 instruct loadUB2L(rRegL dst, memory mem)
6992 %{
6993 match(Set dst (ConvI2L (LoadUB mem)));
6994
6995 ins_cost(125);
6996 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
6997
6998 ins_encode %{
6999 __ movzbq($dst$$Register, $mem$$Address);
7000 %}
7001
7002 ins_pipe(ialu_reg_mem);
7003 %}
7004
7005 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
7006 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7007 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
7008 effect(KILL cr);
7009
7010 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
7011 "andl $dst, right_n_bits($mask, 8)" %}
7012 ins_encode %{
7013 Register Rdst = $dst$$Register;
7014 __ movzbq(Rdst, $mem$$Address);
7015 __ andl(Rdst, $mask$$constant & right_n_bits(8));
7016 %}
7017 ins_pipe(ialu_reg_mem);
7018 %}
7019
7020 // Load Short (16 bit signed)
7021 instruct loadS(rRegI dst, memory mem)
7022 %{
7023 match(Set dst (LoadS mem));
7024
7025 ins_cost(125);
7026 format %{ "movswl $dst, $mem\t# short" %}
7027
7028 ins_encode %{
7029 __ movswl($dst$$Register, $mem$$Address);
7030 %}
7031
7032 ins_pipe(ialu_reg_mem);
7033 %}
7034
7035 // Load Short (16 bit signed) to Byte (8 bit signed)
7036 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7037 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
7038
7039 ins_cost(125);
7040 format %{ "movsbl $dst, $mem\t# short -> byte" %}
7041 ins_encode %{
7042 __ movsbl($dst$$Register, $mem$$Address);
7043 %}
7044 ins_pipe(ialu_reg_mem);
7045 %}
7046
7047 // Load Short (16 bit signed) into Long Register
7048 instruct loadS2L(rRegL dst, memory mem)
7049 %{
7050 match(Set dst (ConvI2L (LoadS mem)));
7051
7052 ins_cost(125);
7053 format %{ "movswq $dst, $mem\t# short -> long" %}
7054
7055 ins_encode %{
7056 __ movswq($dst$$Register, $mem$$Address);
7057 %}
7058
7059 ins_pipe(ialu_reg_mem);
7060 %}
7061
7062 // Load Unsigned Short/Char (16 bit UNsigned)
7063 instruct loadUS(rRegI dst, memory mem)
7064 %{
7065 match(Set dst (LoadUS mem));
7066
7067 ins_cost(125);
7068 format %{ "movzwl $dst, $mem\t# ushort/char" %}
7069
7070 ins_encode %{
7071 __ movzwl($dst$$Register, $mem$$Address);
7072 %}
7073
7074 ins_pipe(ialu_reg_mem);
7075 %}
7076
7077 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
7078 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7079 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
7080
7081 ins_cost(125);
7082 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
7083 ins_encode %{
7084 __ movsbl($dst$$Register, $mem$$Address);
7085 %}
7086 ins_pipe(ialu_reg_mem);
7087 %}
7088
7089 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
7090 instruct loadUS2L(rRegL dst, memory mem)
7091 %{
7092 match(Set dst (ConvI2L (LoadUS mem)));
7093
7094 ins_cost(125);
7095 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
7096
7097 ins_encode %{
7098 __ movzwq($dst$$Register, $mem$$Address);
7099 %}
7100
7101 ins_pipe(ialu_reg_mem);
7102 %}
7103
7104 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
7105 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7106 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7107
7108 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
7109 ins_encode %{
7110 __ movzbq($dst$$Register, $mem$$Address);
7111 %}
7112 ins_pipe(ialu_reg_mem);
7113 %}
7114
7115 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
7116 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7117 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7118 effect(KILL cr);
7119
7120 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
7121 "andl $dst, right_n_bits($mask, 16)" %}
7122 ins_encode %{
7123 Register Rdst = $dst$$Register;
7124 __ movzwq(Rdst, $mem$$Address);
7125 __ andl(Rdst, $mask$$constant & right_n_bits(16));
7126 %}
7127 ins_pipe(ialu_reg_mem);
7128 %}
7129
7130 // Load Integer
7131 instruct loadI(rRegI dst, memory mem)
7132 %{
7133 match(Set dst (LoadI mem));
7134
7135 ins_cost(125);
7136 format %{ "movl $dst, $mem\t# int" %}
7137
7138 ins_encode %{
7139 __ movl($dst$$Register, $mem$$Address);
7140 %}
7141
7142 ins_pipe(ialu_reg_mem);
7143 %}
7144
7145 // Load Integer (32 bit signed) to Byte (8 bit signed)
7146 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7147 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
7148
7149 ins_cost(125);
7150 format %{ "movsbl $dst, $mem\t# int -> byte" %}
7151 ins_encode %{
7152 __ movsbl($dst$$Register, $mem$$Address);
7153 %}
7154 ins_pipe(ialu_reg_mem);
7155 %}
7156
7157 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
7158 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
7159 match(Set dst (AndI (LoadI mem) mask));
7160
7161 ins_cost(125);
7162 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
7163 ins_encode %{
7164 __ movzbl($dst$$Register, $mem$$Address);
7165 %}
7166 ins_pipe(ialu_reg_mem);
7167 %}
7168
7169 // Load Integer (32 bit signed) to Short (16 bit signed)
7170 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
7171 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
7172
7173 ins_cost(125);
7174 format %{ "movswl $dst, $mem\t# int -> short" %}
7175 ins_encode %{
7176 __ movswl($dst$$Register, $mem$$Address);
7177 %}
7178 ins_pipe(ialu_reg_mem);
7179 %}
7180
7181 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
7182 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
7183 match(Set dst (AndI (LoadI mem) mask));
7184
7185 ins_cost(125);
7186 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
7187 ins_encode %{
7188 __ movzwl($dst$$Register, $mem$$Address);
7189 %}
7190 ins_pipe(ialu_reg_mem);
7191 %}
7192
7193 // Load Integer into Long Register
7194 instruct loadI2L(rRegL dst, memory mem)
7195 %{
7196 match(Set dst (ConvI2L (LoadI mem)));
7197
7198 ins_cost(125);
7199 format %{ "movslq $dst, $mem\t# int -> long" %}
7200
7201 ins_encode %{
7202 __ movslq($dst$$Register, $mem$$Address);
7203 %}
7204
7205 ins_pipe(ialu_reg_mem);
7206 %}
7207
7208 // Load Integer with mask 0xFF into Long Register
7209 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7210 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7211
7212 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
7213 ins_encode %{
7214 __ movzbq($dst$$Register, $mem$$Address);
7215 %}
7216 ins_pipe(ialu_reg_mem);
7217 %}
7218
7219 // Load Integer with mask 0xFFFF into Long Register
7220 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
7221 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7222
7223 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
7224 ins_encode %{
7225 __ movzwq($dst$$Register, $mem$$Address);
7226 %}
7227 ins_pipe(ialu_reg_mem);
7228 %}
7229
7230 // Load Integer with a 31-bit mask into Long Register
7231 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
7232 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7233 effect(KILL cr);
7234
7235 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
7236 "andl $dst, $mask" %}
7237 ins_encode %{
7238 Register Rdst = $dst$$Register;
7239 __ movl(Rdst, $mem$$Address);
7240 __ andl(Rdst, $mask$$constant);
7241 %}
7242 ins_pipe(ialu_reg_mem);
7243 %}
7244
7245 // Load Unsigned Integer into Long Register
7246 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
7247 %{
7248 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7249
7250 ins_cost(125);
7251 format %{ "movl $dst, $mem\t# uint -> long" %}
7252
7253 ins_encode %{
7254 __ movl($dst$$Register, $mem$$Address);
7255 %}
7256
7257 ins_pipe(ialu_reg_mem);
7258 %}
7259
7260 // Load Long
7261 instruct loadL(rRegL dst, memory mem)
7262 %{
7263 match(Set dst (LoadL mem));
7264
7265 ins_cost(125);
7266 format %{ "movq $dst, $mem\t# long" %}
7267
7268 ins_encode %{
7269 __ movq($dst$$Register, $mem$$Address);
7270 %}
7271
7272 ins_pipe(ialu_reg_mem); // XXX
7273 %}
7274
7275 // Load Range
7276 instruct loadRange(rRegI dst, memory mem)
7277 %{
7278 match(Set dst (LoadRange mem));
7279
7280 ins_cost(125); // XXX
7281 format %{ "movl $dst, $mem\t# range" %}
7282 ins_encode %{
7283 __ movl($dst$$Register, $mem$$Address);
7284 %}
7285 ins_pipe(ialu_reg_mem);
7286 %}
7287
7288 // Load Pointer
7289 instruct loadP(rRegP dst, memory mem)
7290 %{
7291 match(Set dst (LoadP mem));
7292 predicate(n->as_Load()->barrier_data() == 0);
7293
7294 ins_cost(125); // XXX
7295 format %{ "movq $dst, $mem\t# ptr" %}
7296 ins_encode %{
7297 __ movq($dst$$Register, $mem$$Address);
7298 %}
7299 ins_pipe(ialu_reg_mem); // XXX
7300 %}
7301
7302 // Load Compressed Pointer
7303 instruct loadN(rRegN dst, memory mem)
7304 %{
7305 predicate(n->as_Load()->barrier_data() == 0);
7306 match(Set dst (LoadN mem));
7307
7308 ins_cost(125); // XXX
7309 format %{ "movl $dst, $mem\t# compressed ptr" %}
7310 ins_encode %{
7311 __ movl($dst$$Register, $mem$$Address);
7312 %}
7313 ins_pipe(ialu_reg_mem); // XXX
7314 %}
7315
7316
7317 // Load Klass Pointer
7318 instruct loadKlass(rRegP dst, memory mem)
7319 %{
7320 match(Set dst (LoadKlass mem));
7321
7322 ins_cost(125); // XXX
7323 format %{ "movq $dst, $mem\t# class" %}
7324 ins_encode %{
7325 __ movq($dst$$Register, $mem$$Address);
7326 %}
7327 ins_pipe(ialu_reg_mem); // XXX
7328 %}
7329
7330 // Load narrow Klass Pointer
7331 instruct loadNKlass(rRegN dst, memory mem)
7332 %{
7333 predicate(!UseCompactObjectHeaders);
7334 match(Set dst (LoadNKlass mem));
7335
7336 ins_cost(125); // XXX
7337 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
7338 ins_encode %{
7339 __ movl($dst$$Register, $mem$$Address);
7340 %}
7341 ins_pipe(ialu_reg_mem); // XXX
7342 %}
7343
7344 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
7345 %{
7346 predicate(UseCompactObjectHeaders);
7347 match(Set dst (LoadNKlass mem));
7348 effect(KILL cr);
7349 ins_cost(125);
7350 format %{
7351 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
7352 "shrl $dst, markWord::klass_shift_at_offset"
7353 %}
7354 ins_encode %{
7355 __ movl($dst$$Register, $mem$$Address);
7356 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
7357 %}
7358 ins_pipe(ialu_reg_mem);
7359 %}
7360
7361 // Load Float
7362 instruct loadF(regF dst, memory mem)
7363 %{
7364 match(Set dst (LoadF mem));
7365
7366 ins_cost(145); // XXX
7367 format %{ "movss $dst, $mem\t# float" %}
7368 ins_encode %{
7369 __ movflt($dst$$XMMRegister, $mem$$Address);
7370 %}
7371 ins_pipe(pipe_slow); // XXX
7372 %}
7373
7374 // Load Double
7375 instruct loadD_partial(regD dst, memory mem)
7376 %{
7377 predicate(!UseXmmLoadAndClearUpper);
7378 match(Set dst (LoadD mem));
7379
7380 ins_cost(145); // XXX
7381 format %{ "movlpd $dst, $mem\t# double" %}
7382 ins_encode %{
7383 __ movdbl($dst$$XMMRegister, $mem$$Address);
7384 %}
7385 ins_pipe(pipe_slow); // XXX
7386 %}
7387
7388 instruct loadD(regD dst, memory mem)
7389 %{
7390 predicate(UseXmmLoadAndClearUpper);
7391 match(Set dst (LoadD mem));
7392
7393 ins_cost(145); // XXX
7394 format %{ "movsd $dst, $mem\t# double" %}
7395 ins_encode %{
7396 __ movdbl($dst$$XMMRegister, $mem$$Address);
7397 %}
7398 ins_pipe(pipe_slow); // XXX
7399 %}
7400
7401 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
7402 %{
7403 match(Set dst con);
7404
7405 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
7406
7407 ins_encode %{
7408 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7409 %}
7410
7411 ins_pipe(ialu_reg_fat);
7412 %}
7413
7414 // min = java.lang.Math.min(float a, float b)
7415 // max = java.lang.Math.max(float a, float b)
7416 instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
7417 %{
7418 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7419 match(Set dst (MaxF a b));
7420 match(Set dst (MinF a b));
7421
7422 format %{ "minmaxF $dst, $a, $b" %}
7423 ins_encode %{
7424 int opcode = this->ideal_Opcode();
7425 __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7426 %}
7427 ins_pipe( pipe_slow );
7428 %}
7429
7430 instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, rRegI rtmp, rFlagsReg cr)
7431 %{
7432 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7433 match(Set dst (MaxF a b));
7434 match(Set dst (MinF a b));
7435 effect(USE a, USE b, TEMP rtmp, KILL cr);
7436
7437 format %{ "minmaxF_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7438 ins_encode %{
7439 int opcode = this->ideal_Opcode();
7440 bool min = (opcode == Op_MinF) ? true : false;
7441 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7442 min, fp_prec_flt /*pt*/);
7443 %}
7444 ins_pipe( pipe_slow );
7445 %}
7446
7447 // min = java.lang.Math.min(float a, float b)
7448 // max = java.lang.Math.max(float a, float b)
7449 instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
7450 %{
7451 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7452 match(Set dst (MaxF a b));
7453 match(Set dst (MinF a b));
7454 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
7455
7456 format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7457 ins_encode %{
7458 int opcode = this->ideal_Opcode();
7459 int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
7460 __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7461 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7462 %}
7463 ins_pipe( pipe_slow );
7464 %}
7465
7466 instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, rRegI rtmp, rFlagsReg cr)
7467 %{
7468 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7469 match(Set dst (MaxF a b));
7470 match(Set dst (MinF a b));
7471 effect(USE a, USE b, TEMP rtmp, KILL cr);
7472
7473 format %{ "minmaxF_reduction $dst, $a, $b \t!using $rtmp as TEMP" %}
7474 ins_encode %{
7475 int opcode = this->ideal_Opcode();
7476 bool min = (opcode == Op_MinF) ? true : false;
7477 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7478 min, fp_prec_flt /*pt*/);
7479 %}
7480 ins_pipe( pipe_slow );
7481 %}
7482
7483 // min = java.lang.Math.min(double a, double b)
7484 // max = java.lang.Math.max(double a, double b)
7485 instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
7486 %{
7487 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7488 match(Set dst (MaxD a b));
7489 match(Set dst (MinD a b));
7490
7491 format %{ "minmaxD $dst, $a, $b" %}
7492 ins_encode %{
7493 int opcode = this->ideal_Opcode();
7494 __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7495 %}
7496 ins_pipe( pipe_slow );
7497 %}
7498
7499 instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, rRegI rtmp, rFlagsReg cr)
7500 %{
7501 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7502 match(Set dst (MaxD a b));
7503 match(Set dst (MinD a b));
7504 effect(USE a, USE b, TEMP rtmp, KILL cr);
7505
7506 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7507 ins_encode %{
7508 int opcode = this->ideal_Opcode();
7509 bool min = (opcode == Op_MinD) ? true : false;
7510 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7511 min, fp_prec_dbl /*pt*/);
7512 %}
7513 ins_pipe( pipe_slow );
7514 %}
7515
7516 // min = java.lang.Math.min(double a, double b)
7517 // max = java.lang.Math.max(double a, double b)
7518 instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
7519 %{
7520 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7521 match(Set dst (MaxD a b));
7522 match(Set dst (MinD a b));
7523 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
7524
7525 format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7526 ins_encode %{
7527 int opcode = this->ideal_Opcode();
7528 int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
7529 __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7530 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7531 %}
7532 ins_pipe( pipe_slow );
7533 %}
7534
7535 instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, rRegL rtmp, rFlagsReg cr)
7536 %{
7537 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7538 match(Set dst (MaxD a b));
7539 match(Set dst (MinD a b));
7540 effect(USE a, USE b, TEMP rtmp, KILL cr);
7541
7542 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7543 ins_encode %{
7544 int opcode = this->ideal_Opcode();
7545 bool min = (opcode == Op_MinD) ? true : false;
7546 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7547 min, fp_prec_dbl /*pt*/);
7548 %}
7549 ins_pipe( pipe_slow );
7550 %}
7551
7552 // Load Effective Address
7553 instruct leaP8(rRegP dst, indOffset8 mem)
7554 %{
7555 match(Set dst mem);
7556
7557 ins_cost(110); // XXX
7558 format %{ "leaq $dst, $mem\t# ptr 8" %}
7559 ins_encode %{
7560 __ leaq($dst$$Register, $mem$$Address);
7561 %}
7562 ins_pipe(ialu_reg_reg_fat);
7563 %}
7564
7565 instruct leaP32(rRegP dst, indOffset32 mem)
7566 %{
7567 match(Set dst mem);
7568
7569 ins_cost(110);
7570 format %{ "leaq $dst, $mem\t# ptr 32" %}
7571 ins_encode %{
7572 __ leaq($dst$$Register, $mem$$Address);
7573 %}
7574 ins_pipe(ialu_reg_reg_fat);
7575 %}
7576
7577 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
7578 %{
7579 match(Set dst mem);
7580
7581 ins_cost(110);
7582 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
7583 ins_encode %{
7584 __ leaq($dst$$Register, $mem$$Address);
7585 %}
7586 ins_pipe(ialu_reg_reg_fat);
7587 %}
7588
7589 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
7590 %{
7591 match(Set dst mem);
7592
7593 ins_cost(110);
7594 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7595 ins_encode %{
7596 __ leaq($dst$$Register, $mem$$Address);
7597 %}
7598 ins_pipe(ialu_reg_reg_fat);
7599 %}
7600
7601 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
7602 %{
7603 match(Set dst mem);
7604
7605 ins_cost(110);
7606 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7607 ins_encode %{
7608 __ leaq($dst$$Register, $mem$$Address);
7609 %}
7610 ins_pipe(ialu_reg_reg_fat);
7611 %}
7612
7613 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
7614 %{
7615 match(Set dst mem);
7616
7617 ins_cost(110);
7618 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
7619 ins_encode %{
7620 __ leaq($dst$$Register, $mem$$Address);
7621 %}
7622 ins_pipe(ialu_reg_reg_fat);
7623 %}
7624
7625 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
7626 %{
7627 match(Set dst mem);
7628
7629 ins_cost(110);
7630 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
7631 ins_encode %{
7632 __ leaq($dst$$Register, $mem$$Address);
7633 %}
7634 ins_pipe(ialu_reg_reg_fat);
7635 %}
7636
7637 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
7638 %{
7639 match(Set dst mem);
7640
7641 ins_cost(110);
7642 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
7643 ins_encode %{
7644 __ leaq($dst$$Register, $mem$$Address);
7645 %}
7646 ins_pipe(ialu_reg_reg_fat);
7647 %}
7648
7649 // Load Effective Address which uses Narrow (32-bits) oop
7650 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
7651 %{
7652 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
7653 match(Set dst mem);
7654
7655 ins_cost(110);
7656 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
7657 ins_encode %{
7658 __ leaq($dst$$Register, $mem$$Address);
7659 %}
7660 ins_pipe(ialu_reg_reg_fat);
7661 %}
7662
7663 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
7664 %{
7665 predicate(CompressedOops::shift() == 0);
7666 match(Set dst mem);
7667
7668 ins_cost(110); // XXX
7669 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
7670 ins_encode %{
7671 __ leaq($dst$$Register, $mem$$Address);
7672 %}
7673 ins_pipe(ialu_reg_reg_fat);
7674 %}
7675
7676 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
7677 %{
7678 predicate(CompressedOops::shift() == 0);
7679 match(Set dst mem);
7680
7681 ins_cost(110);
7682 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
7683 ins_encode %{
7684 __ leaq($dst$$Register, $mem$$Address);
7685 %}
7686 ins_pipe(ialu_reg_reg_fat);
7687 %}
7688
7689 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
7690 %{
7691 predicate(CompressedOops::shift() == 0);
7692 match(Set dst mem);
7693
7694 ins_cost(110);
7695 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
7696 ins_encode %{
7697 __ leaq($dst$$Register, $mem$$Address);
7698 %}
7699 ins_pipe(ialu_reg_reg_fat);
7700 %}
7701
7702 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
7703 %{
7704 predicate(CompressedOops::shift() == 0);
7705 match(Set dst mem);
7706
7707 ins_cost(110);
7708 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
7709 ins_encode %{
7710 __ leaq($dst$$Register, $mem$$Address);
7711 %}
7712 ins_pipe(ialu_reg_reg_fat);
7713 %}
7714
7715 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
7716 %{
7717 predicate(CompressedOops::shift() == 0);
7718 match(Set dst mem);
7719
7720 ins_cost(110);
7721 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
7722 ins_encode %{
7723 __ leaq($dst$$Register, $mem$$Address);
7724 %}
7725 ins_pipe(ialu_reg_reg_fat);
7726 %}
7727
7728 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
7729 %{
7730 predicate(CompressedOops::shift() == 0);
7731 match(Set dst mem);
7732
7733 ins_cost(110);
7734 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
7735 ins_encode %{
7736 __ leaq($dst$$Register, $mem$$Address);
7737 %}
7738 ins_pipe(ialu_reg_reg_fat);
7739 %}
7740
7741 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
7742 %{
7743 predicate(CompressedOops::shift() == 0);
7744 match(Set dst mem);
7745
7746 ins_cost(110);
7747 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
7748 ins_encode %{
7749 __ leaq($dst$$Register, $mem$$Address);
7750 %}
7751 ins_pipe(ialu_reg_reg_fat);
7752 %}
7753
7754 instruct loadConI(rRegI dst, immI src)
7755 %{
7756 match(Set dst src);
7757
7758 format %{ "movl $dst, $src\t# int" %}
7759 ins_encode %{
7760 __ movl($dst$$Register, $src$$constant);
7761 %}
7762 ins_pipe(ialu_reg_fat); // XXX
7763 %}
7764
7765 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
7766 %{
7767 match(Set dst src);
7768 effect(KILL cr);
7769
7770 ins_cost(50);
7771 format %{ "xorl $dst, $dst\t# int" %}
7772 ins_encode %{
7773 __ xorl($dst$$Register, $dst$$Register);
7774 %}
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct loadConL(rRegL dst, immL src)
7779 %{
7780 match(Set dst src);
7781
7782 ins_cost(150);
7783 format %{ "movq $dst, $src\t# long" %}
7784 ins_encode %{
7785 __ mov64($dst$$Register, $src$$constant);
7786 %}
7787 ins_pipe(ialu_reg);
7788 %}
7789
7790 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
7791 %{
7792 match(Set dst src);
7793 effect(KILL cr);
7794
7795 ins_cost(50);
7796 format %{ "xorl $dst, $dst\t# long" %}
7797 ins_encode %{
7798 __ xorl($dst$$Register, $dst$$Register);
7799 %}
7800 ins_pipe(ialu_reg); // XXX
7801 %}
7802
7803 instruct loadConUL32(rRegL dst, immUL32 src)
7804 %{
7805 match(Set dst src);
7806
7807 ins_cost(60);
7808 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
7809 ins_encode %{
7810 __ movl($dst$$Register, $src$$constant);
7811 %}
7812 ins_pipe(ialu_reg);
7813 %}
7814
7815 instruct loadConL32(rRegL dst, immL32 src)
7816 %{
7817 match(Set dst src);
7818
7819 ins_cost(70);
7820 format %{ "movq $dst, $src\t# long (32-bit)" %}
7821 ins_encode %{
7822 __ movq($dst$$Register, $src$$constant);
7823 %}
7824 ins_pipe(ialu_reg);
7825 %}
7826
7827 instruct loadConP(rRegP dst, immP con) %{
7828 match(Set dst con);
7829
7830 format %{ "movq $dst, $con\t# ptr" %}
7831 ins_encode %{
7832 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
7833 %}
7834 ins_pipe(ialu_reg_fat); // XXX
7835 %}
7836
7837 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
7838 %{
7839 match(Set dst src);
7840 effect(KILL cr);
7841
7842 ins_cost(50);
7843 format %{ "xorl $dst, $dst\t# ptr" %}
7844 ins_encode %{
7845 __ xorl($dst$$Register, $dst$$Register);
7846 %}
7847 ins_pipe(ialu_reg);
7848 %}
7849
7850 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
7851 %{
7852 match(Set dst src);
7853 effect(KILL cr);
7854
7855 ins_cost(60);
7856 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
7857 ins_encode %{
7858 __ movl($dst$$Register, $src$$constant);
7859 %}
7860 ins_pipe(ialu_reg);
7861 %}
7862
7863 instruct loadConF(regF dst, immF con) %{
7864 match(Set dst con);
7865 ins_cost(125);
7866 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
7867 ins_encode %{
7868 __ movflt($dst$$XMMRegister, $constantaddress($con));
7869 %}
7870 ins_pipe(pipe_slow);
7871 %}
7872
7873 instruct loadConH(regF dst, immH con) %{
7874 match(Set dst con);
7875 ins_cost(125);
7876 format %{ "movss $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
7877 ins_encode %{
7878 __ movflt($dst$$XMMRegister, $constantaddress($con));
7879 %}
7880 ins_pipe(pipe_slow);
7881 %}
7882
7883 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
7884 match(Set dst src);
7885 effect(KILL cr);
7886 format %{ "xorq $dst, $src\t# compressed null pointer" %}
7887 ins_encode %{
7888 __ xorq($dst$$Register, $dst$$Register);
7889 %}
7890 ins_pipe(ialu_reg);
7891 %}
7892
7893 instruct loadConN(rRegN dst, immN src) %{
7894 match(Set dst src);
7895
7896 ins_cost(125);
7897 format %{ "movl $dst, $src\t# compressed ptr" %}
7898 ins_encode %{
7899 address con = (address)$src$$constant;
7900 if (con == nullptr) {
7901 ShouldNotReachHere();
7902 } else {
7903 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
7904 }
7905 %}
7906 ins_pipe(ialu_reg_fat); // XXX
7907 %}
7908
7909 instruct loadConNKlass(rRegN dst, immNKlass src) %{
7910 match(Set dst src);
7911
7912 ins_cost(125);
7913 format %{ "movl $dst, $src\t# compressed klass ptr" %}
7914 ins_encode %{
7915 address con = (address)$src$$constant;
7916 if (con == nullptr) {
7917 ShouldNotReachHere();
7918 } else {
7919 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
7920 }
7921 %}
7922 ins_pipe(ialu_reg_fat); // XXX
7923 %}
7924
7925 instruct loadConF0(regF dst, immF0 src)
7926 %{
7927 match(Set dst src);
7928 ins_cost(100);
7929
7930 format %{ "xorps $dst, $dst\t# float 0.0" %}
7931 ins_encode %{
7932 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7933 %}
7934 ins_pipe(pipe_slow);
7935 %}
7936
7937 // Use the same format since predicate() can not be used here.
7938 instruct loadConD(regD dst, immD con) %{
7939 match(Set dst con);
7940 ins_cost(125);
7941 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
7942 ins_encode %{
7943 __ movdbl($dst$$XMMRegister, $constantaddress($con));
7944 %}
7945 ins_pipe(pipe_slow);
7946 %}
7947
7948 instruct loadConD0(regD dst, immD0 src)
7949 %{
7950 match(Set dst src);
7951 ins_cost(100);
7952
7953 format %{ "xorpd $dst, $dst\t# double 0.0" %}
7954 ins_encode %{
7955 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
7956 %}
7957 ins_pipe(pipe_slow);
7958 %}
7959
7960 instruct loadSSI(rRegI dst, stackSlotI src)
7961 %{
7962 match(Set dst src);
7963
7964 ins_cost(125);
7965 format %{ "movl $dst, $src\t# int stk" %}
7966 ins_encode %{
7967 __ movl($dst$$Register, $src$$Address);
7968 %}
7969 ins_pipe(ialu_reg_mem);
7970 %}
7971
7972 instruct loadSSL(rRegL dst, stackSlotL src)
7973 %{
7974 match(Set dst src);
7975
7976 ins_cost(125);
7977 format %{ "movq $dst, $src\t# long stk" %}
7978 ins_encode %{
7979 __ movq($dst$$Register, $src$$Address);
7980 %}
7981 ins_pipe(ialu_reg_mem);
7982 %}
7983
7984 instruct loadSSP(rRegP dst, stackSlotP src)
7985 %{
7986 match(Set dst src);
7987
7988 ins_cost(125);
7989 format %{ "movq $dst, $src\t# ptr stk" %}
7990 ins_encode %{
7991 __ movq($dst$$Register, $src$$Address);
7992 %}
7993 ins_pipe(ialu_reg_mem);
7994 %}
7995
7996 instruct loadSSF(regF dst, stackSlotF src)
7997 %{
7998 match(Set dst src);
7999
8000 ins_cost(125);
8001 format %{ "movss $dst, $src\t# float stk" %}
8002 ins_encode %{
8003 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
8004 %}
8005 ins_pipe(pipe_slow); // XXX
8006 %}
8007
8008 // Use the same format since predicate() can not be used here.
8009 instruct loadSSD(regD dst, stackSlotD src)
8010 %{
8011 match(Set dst src);
8012
8013 ins_cost(125);
8014 format %{ "movsd $dst, $src\t# double stk" %}
8015 ins_encode %{
8016 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
8017 %}
8018 ins_pipe(pipe_slow); // XXX
8019 %}
8020
8021 // Prefetch instructions for allocation.
8022 // Must be safe to execute with invalid address (cannot fault).
8023
8024 instruct prefetchAlloc( memory mem ) %{
8025 predicate(AllocatePrefetchInstr==3);
8026 match(PrefetchAllocation mem);
8027 ins_cost(125);
8028
8029 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
8030 ins_encode %{
8031 __ prefetchw($mem$$Address);
8032 %}
8033 ins_pipe(ialu_mem);
8034 %}
8035
8036 instruct prefetchAllocNTA( memory mem ) %{
8037 predicate(AllocatePrefetchInstr==0);
8038 match(PrefetchAllocation mem);
8039 ins_cost(125);
8040
8041 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
8042 ins_encode %{
8043 __ prefetchnta($mem$$Address);
8044 %}
8045 ins_pipe(ialu_mem);
8046 %}
8047
8048 instruct prefetchAllocT0( memory mem ) %{
8049 predicate(AllocatePrefetchInstr==1);
8050 match(PrefetchAllocation mem);
8051 ins_cost(125);
8052
8053 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
8054 ins_encode %{
8055 __ prefetcht0($mem$$Address);
8056 %}
8057 ins_pipe(ialu_mem);
8058 %}
8059
8060 instruct prefetchAllocT2( memory mem ) %{
8061 predicate(AllocatePrefetchInstr==2);
8062 match(PrefetchAllocation mem);
8063 ins_cost(125);
8064
8065 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
8066 ins_encode %{
8067 __ prefetcht2($mem$$Address);
8068 %}
8069 ins_pipe(ialu_mem);
8070 %}
8071
8072 //----------Store Instructions-------------------------------------------------
8073
8074 // Store Byte
8075 instruct storeB(memory mem, rRegI src)
8076 %{
8077 match(Set mem (StoreB mem src));
8078
8079 ins_cost(125); // XXX
8080 format %{ "movb $mem, $src\t# byte" %}
8081 ins_encode %{
8082 __ movb($mem$$Address, $src$$Register);
8083 %}
8084 ins_pipe(ialu_mem_reg);
8085 %}
8086
8087 // Store Char/Short
8088 instruct storeC(memory mem, rRegI src)
8089 %{
8090 match(Set mem (StoreC mem src));
8091
8092 ins_cost(125); // XXX
8093 format %{ "movw $mem, $src\t# char/short" %}
8094 ins_encode %{
8095 __ movw($mem$$Address, $src$$Register);
8096 %}
8097 ins_pipe(ialu_mem_reg);
8098 %}
8099
8100 // Store Integer
8101 instruct storeI(memory mem, rRegI src)
8102 %{
8103 match(Set mem (StoreI mem src));
8104
8105 ins_cost(125); // XXX
8106 format %{ "movl $mem, $src\t# int" %}
8107 ins_encode %{
8108 __ movl($mem$$Address, $src$$Register);
8109 %}
8110 ins_pipe(ialu_mem_reg);
8111 %}
8112
8113 // Store Long
8114 instruct storeL(memory mem, rRegL src)
8115 %{
8116 match(Set mem (StoreL mem src));
8117
8118 ins_cost(125); // XXX
8119 format %{ "movq $mem, $src\t# long" %}
8120 ins_encode %{
8121 __ movq($mem$$Address, $src$$Register);
8122 %}
8123 ins_pipe(ialu_mem_reg); // XXX
8124 %}
8125
8126 // Store Pointer
8127 instruct storeP(memory mem, any_RegP src)
8128 %{
8129 predicate(n->as_Store()->barrier_data() == 0);
8130 match(Set mem (StoreP mem src));
8131
8132 ins_cost(125); // XXX
8133 format %{ "movq $mem, $src\t# ptr" %}
8134 ins_encode %{
8135 __ movq($mem$$Address, $src$$Register);
8136 %}
8137 ins_pipe(ialu_mem_reg);
8138 %}
8139
8140 instruct storeImmP0(memory mem, immP0 zero)
8141 %{
8142 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
8143 match(Set mem (StoreP mem zero));
8144
8145 ins_cost(125); // XXX
8146 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
8147 ins_encode %{
8148 __ movq($mem$$Address, r12);
8149 %}
8150 ins_pipe(ialu_mem_reg);
8151 %}
8152
8153 // Store Null Pointer, mark word, or other simple pointer constant.
8154 instruct storeImmP(memory mem, immP31 src)
8155 %{
8156 predicate(n->as_Store()->barrier_data() == 0);
8157 match(Set mem (StoreP mem src));
8158
8159 ins_cost(150); // XXX
8160 format %{ "movq $mem, $src\t# ptr" %}
8161 ins_encode %{
8162 __ movq($mem$$Address, $src$$constant);
8163 %}
8164 ins_pipe(ialu_mem_imm);
8165 %}
8166
8167 // Store Compressed Pointer
8168 instruct storeN(memory mem, rRegN src)
8169 %{
8170 predicate(n->as_Store()->barrier_data() == 0);
8171 match(Set mem (StoreN mem src));
8172
8173 ins_cost(125); // XXX
8174 format %{ "movl $mem, $src\t# compressed ptr" %}
8175 ins_encode %{
8176 __ movl($mem$$Address, $src$$Register);
8177 %}
8178 ins_pipe(ialu_mem_reg);
8179 %}
8180
8181 instruct storeNKlass(memory mem, rRegN src)
8182 %{
8183 match(Set mem (StoreNKlass mem src));
8184
8185 ins_cost(125); // XXX
8186 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8187 ins_encode %{
8188 __ movl($mem$$Address, $src$$Register);
8189 %}
8190 ins_pipe(ialu_mem_reg);
8191 %}
8192
8193 instruct storeImmN0(memory mem, immN0 zero)
8194 %{
8195 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
8196 match(Set mem (StoreN mem zero));
8197
8198 ins_cost(125); // XXX
8199 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
8200 ins_encode %{
8201 __ movl($mem$$Address, r12);
8202 %}
8203 ins_pipe(ialu_mem_reg);
8204 %}
8205
8206 instruct storeImmN(memory mem, immN src)
8207 %{
8208 predicate(n->as_Store()->barrier_data() == 0);
8209 match(Set mem (StoreN mem src));
8210
8211 ins_cost(150); // XXX
8212 format %{ "movl $mem, $src\t# compressed ptr" %}
8213 ins_encode %{
8214 address con = (address)$src$$constant;
8215 if (con == nullptr) {
8216 __ movl($mem$$Address, 0);
8217 } else {
8218 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
8219 }
8220 %}
8221 ins_pipe(ialu_mem_imm);
8222 %}
8223
8224 instruct storeImmNKlass(memory mem, immNKlass src)
8225 %{
8226 match(Set mem (StoreNKlass mem src));
8227
8228 ins_cost(150); // XXX
8229 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8230 ins_encode %{
8231 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
8232 %}
8233 ins_pipe(ialu_mem_imm);
8234 %}
8235
8236 // Store Integer Immediate
8237 instruct storeImmI0(memory mem, immI_0 zero)
8238 %{
8239 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8240 match(Set mem (StoreI mem zero));
8241
8242 ins_cost(125); // XXX
8243 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
8244 ins_encode %{
8245 __ movl($mem$$Address, r12);
8246 %}
8247 ins_pipe(ialu_mem_reg);
8248 %}
8249
8250 instruct storeImmI(memory mem, immI src)
8251 %{
8252 match(Set mem (StoreI mem src));
8253
8254 ins_cost(150);
8255 format %{ "movl $mem, $src\t# int" %}
8256 ins_encode %{
8257 __ movl($mem$$Address, $src$$constant);
8258 %}
8259 ins_pipe(ialu_mem_imm);
8260 %}
8261
8262 // Store Long Immediate
8263 instruct storeImmL0(memory mem, immL0 zero)
8264 %{
8265 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8266 match(Set mem (StoreL mem zero));
8267
8268 ins_cost(125); // XXX
8269 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
8270 ins_encode %{
8271 __ movq($mem$$Address, r12);
8272 %}
8273 ins_pipe(ialu_mem_reg);
8274 %}
8275
8276 instruct storeImmL(memory mem, immL32 src)
8277 %{
8278 match(Set mem (StoreL mem src));
8279
8280 ins_cost(150);
8281 format %{ "movq $mem, $src\t# long" %}
8282 ins_encode %{
8283 __ movq($mem$$Address, $src$$constant);
8284 %}
8285 ins_pipe(ialu_mem_imm);
8286 %}
8287
8288 // Store Short/Char Immediate
8289 instruct storeImmC0(memory mem, immI_0 zero)
8290 %{
8291 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8292 match(Set mem (StoreC mem zero));
8293
8294 ins_cost(125); // XXX
8295 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
8296 ins_encode %{
8297 __ movw($mem$$Address, r12);
8298 %}
8299 ins_pipe(ialu_mem_reg);
8300 %}
8301
8302 instruct storeImmI16(memory mem, immI16 src)
8303 %{
8304 predicate(UseStoreImmI16);
8305 match(Set mem (StoreC mem src));
8306
8307 ins_cost(150);
8308 format %{ "movw $mem, $src\t# short/char" %}
8309 ins_encode %{
8310 __ movw($mem$$Address, $src$$constant);
8311 %}
8312 ins_pipe(ialu_mem_imm);
8313 %}
8314
8315 // Store Byte Immediate
8316 instruct storeImmB0(memory mem, immI_0 zero)
8317 %{
8318 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8319 match(Set mem (StoreB mem zero));
8320
8321 ins_cost(125); // XXX
8322 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
8323 ins_encode %{
8324 __ movb($mem$$Address, r12);
8325 %}
8326 ins_pipe(ialu_mem_reg);
8327 %}
8328
8329 instruct storeImmB(memory mem, immI8 src)
8330 %{
8331 match(Set mem (StoreB mem src));
8332
8333 ins_cost(150); // XXX
8334 format %{ "movb $mem, $src\t# byte" %}
8335 ins_encode %{
8336 __ movb($mem$$Address, $src$$constant);
8337 %}
8338 ins_pipe(ialu_mem_imm);
8339 %}
8340
8341 // Store Float
8342 instruct storeF(memory mem, regF src)
8343 %{
8344 match(Set mem (StoreF mem src));
8345
8346 ins_cost(95); // XXX
8347 format %{ "movss $mem, $src\t# float" %}
8348 ins_encode %{
8349 __ movflt($mem$$Address, $src$$XMMRegister);
8350 %}
8351 ins_pipe(pipe_slow); // XXX
8352 %}
8353
8354 // Store immediate Float value (it is faster than store from XMM register)
8355 instruct storeF0(memory mem, immF0 zero)
8356 %{
8357 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8358 match(Set mem (StoreF mem zero));
8359
8360 ins_cost(25); // XXX
8361 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
8362 ins_encode %{
8363 __ movl($mem$$Address, r12);
8364 %}
8365 ins_pipe(ialu_mem_reg);
8366 %}
8367
8368 instruct storeF_imm(memory mem, immF src)
8369 %{
8370 match(Set mem (StoreF mem src));
8371
8372 ins_cost(50);
8373 format %{ "movl $mem, $src\t# float" %}
8374 ins_encode %{
8375 __ movl($mem$$Address, jint_cast($src$$constant));
8376 %}
8377 ins_pipe(ialu_mem_imm);
8378 %}
8379
8380 // Store Double
8381 instruct storeD(memory mem, regD src)
8382 %{
8383 match(Set mem (StoreD mem src));
8384
8385 ins_cost(95); // XXX
8386 format %{ "movsd $mem, $src\t# double" %}
8387 ins_encode %{
8388 __ movdbl($mem$$Address, $src$$XMMRegister);
8389 %}
8390 ins_pipe(pipe_slow); // XXX
8391 %}
8392
8393 // Store immediate double 0.0 (it is faster than store from XMM register)
8394 instruct storeD0_imm(memory mem, immD0 src)
8395 %{
8396 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
8397 match(Set mem (StoreD mem src));
8398
8399 ins_cost(50);
8400 format %{ "movq $mem, $src\t# double 0." %}
8401 ins_encode %{
8402 __ movq($mem$$Address, $src$$constant);
8403 %}
8404 ins_pipe(ialu_mem_imm);
8405 %}
8406
8407 instruct storeD0(memory mem, immD0 zero)
8408 %{
8409 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8410 match(Set mem (StoreD mem zero));
8411
8412 ins_cost(25); // XXX
8413 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
8414 ins_encode %{
8415 __ movq($mem$$Address, r12);
8416 %}
8417 ins_pipe(ialu_mem_reg);
8418 %}
8419
8420 instruct storeSSI(stackSlotI dst, rRegI src)
8421 %{
8422 match(Set dst src);
8423
8424 ins_cost(100);
8425 format %{ "movl $dst, $src\t# int stk" %}
8426 ins_encode %{
8427 __ movl($dst$$Address, $src$$Register);
8428 %}
8429 ins_pipe( ialu_mem_reg );
8430 %}
8431
8432 instruct storeSSL(stackSlotL dst, rRegL src)
8433 %{
8434 match(Set dst src);
8435
8436 ins_cost(100);
8437 format %{ "movq $dst, $src\t# long stk" %}
8438 ins_encode %{
8439 __ movq($dst$$Address, $src$$Register);
8440 %}
8441 ins_pipe(ialu_mem_reg);
8442 %}
8443
8444 instruct storeSSP(stackSlotP dst, rRegP src)
8445 %{
8446 match(Set dst src);
8447
8448 ins_cost(100);
8449 format %{ "movq $dst, $src\t# ptr stk" %}
8450 ins_encode %{
8451 __ movq($dst$$Address, $src$$Register);
8452 %}
8453 ins_pipe(ialu_mem_reg);
8454 %}
8455
8456 instruct storeSSF(stackSlotF dst, regF src)
8457 %{
8458 match(Set dst src);
8459
8460 ins_cost(95); // XXX
8461 format %{ "movss $dst, $src\t# float stk" %}
8462 ins_encode %{
8463 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
8464 %}
8465 ins_pipe(pipe_slow); // XXX
8466 %}
8467
8468 instruct storeSSD(stackSlotD dst, regD src)
8469 %{
8470 match(Set dst src);
8471
8472 ins_cost(95); // XXX
8473 format %{ "movsd $dst, $src\t# double stk" %}
8474 ins_encode %{
8475 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
8476 %}
8477 ins_pipe(pipe_slow); // XXX
8478 %}
8479
8480 instruct cacheWB(indirect addr)
8481 %{
8482 predicate(VM_Version::supports_data_cache_line_flush());
8483 match(CacheWB addr);
8484
8485 ins_cost(100);
8486 format %{"cache wb $addr" %}
8487 ins_encode %{
8488 assert($addr->index_position() < 0, "should be");
8489 assert($addr$$disp == 0, "should be");
8490 __ cache_wb(Address($addr$$base$$Register, 0));
8491 %}
8492 ins_pipe(pipe_slow); // XXX
8493 %}
8494
8495 instruct cacheWBPreSync()
8496 %{
8497 predicate(VM_Version::supports_data_cache_line_flush());
8498 match(CacheWBPreSync);
8499
8500 ins_cost(100);
8501 format %{"cache wb presync" %}
8502 ins_encode %{
8503 __ cache_wbsync(true);
8504 %}
8505 ins_pipe(pipe_slow); // XXX
8506 %}
8507
8508 instruct cacheWBPostSync()
8509 %{
8510 predicate(VM_Version::supports_data_cache_line_flush());
8511 match(CacheWBPostSync);
8512
8513 ins_cost(100);
8514 format %{"cache wb postsync" %}
8515 ins_encode %{
8516 __ cache_wbsync(false);
8517 %}
8518 ins_pipe(pipe_slow); // XXX
8519 %}
8520
8521 //----------BSWAP Instructions-------------------------------------------------
8522 instruct bytes_reverse_int(rRegI dst) %{
8523 match(Set dst (ReverseBytesI dst));
8524
8525 format %{ "bswapl $dst" %}
8526 ins_encode %{
8527 __ bswapl($dst$$Register);
8528 %}
8529 ins_pipe( ialu_reg );
8530 %}
8531
8532 instruct bytes_reverse_long(rRegL dst) %{
8533 match(Set dst (ReverseBytesL dst));
8534
8535 format %{ "bswapq $dst" %}
8536 ins_encode %{
8537 __ bswapq($dst$$Register);
8538 %}
8539 ins_pipe( ialu_reg);
8540 %}
8541
8542 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
8543 match(Set dst (ReverseBytesUS dst));
8544 effect(KILL cr);
8545
8546 format %{ "bswapl $dst\n\t"
8547 "shrl $dst,16\n\t" %}
8548 ins_encode %{
8549 __ bswapl($dst$$Register);
8550 __ shrl($dst$$Register, 16);
8551 %}
8552 ins_pipe( ialu_reg );
8553 %}
8554
8555 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
8556 match(Set dst (ReverseBytesS dst));
8557 effect(KILL cr);
8558
8559 format %{ "bswapl $dst\n\t"
8560 "sar $dst,16\n\t" %}
8561 ins_encode %{
8562 __ bswapl($dst$$Register);
8563 __ sarl($dst$$Register, 16);
8564 %}
8565 ins_pipe( ialu_reg );
8566 %}
8567
8568 //---------- Zeros Count Instructions ------------------------------------------
8569
8570 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8571 predicate(UseCountLeadingZerosInstruction);
8572 match(Set dst (CountLeadingZerosI src));
8573 effect(KILL cr);
8574
8575 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8576 ins_encode %{
8577 __ lzcntl($dst$$Register, $src$$Register);
8578 %}
8579 ins_pipe(ialu_reg);
8580 %}
8581
8582 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8583 predicate(UseCountLeadingZerosInstruction);
8584 match(Set dst (CountLeadingZerosI (LoadI src)));
8585 effect(KILL cr);
8586 ins_cost(175);
8587 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8588 ins_encode %{
8589 __ lzcntl($dst$$Register, $src$$Address);
8590 %}
8591 ins_pipe(ialu_reg_mem);
8592 %}
8593
8594 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
8595 predicate(!UseCountLeadingZerosInstruction);
8596 match(Set dst (CountLeadingZerosI src));
8597 effect(KILL cr);
8598
8599 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
8600 "jnz skip\n\t"
8601 "movl $dst, -1\n"
8602 "skip:\n\t"
8603 "negl $dst\n\t"
8604 "addl $dst, 31" %}
8605 ins_encode %{
8606 Register Rdst = $dst$$Register;
8607 Register Rsrc = $src$$Register;
8608 Label skip;
8609 __ bsrl(Rdst, Rsrc);
8610 __ jccb(Assembler::notZero, skip);
8611 __ movl(Rdst, -1);
8612 __ bind(skip);
8613 __ negl(Rdst);
8614 __ addl(Rdst, BitsPerInt - 1);
8615 %}
8616 ins_pipe(ialu_reg);
8617 %}
8618
8619 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8620 predicate(UseCountLeadingZerosInstruction);
8621 match(Set dst (CountLeadingZerosL src));
8622 effect(KILL cr);
8623
8624 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8625 ins_encode %{
8626 __ lzcntq($dst$$Register, $src$$Register);
8627 %}
8628 ins_pipe(ialu_reg);
8629 %}
8630
8631 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8632 predicate(UseCountLeadingZerosInstruction);
8633 match(Set dst (CountLeadingZerosL (LoadL src)));
8634 effect(KILL cr);
8635 ins_cost(175);
8636 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8637 ins_encode %{
8638 __ lzcntq($dst$$Register, $src$$Address);
8639 %}
8640 ins_pipe(ialu_reg_mem);
8641 %}
8642
8643 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
8644 predicate(!UseCountLeadingZerosInstruction);
8645 match(Set dst (CountLeadingZerosL src));
8646 effect(KILL cr);
8647
8648 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
8649 "jnz skip\n\t"
8650 "movl $dst, -1\n"
8651 "skip:\n\t"
8652 "negl $dst\n\t"
8653 "addl $dst, 63" %}
8654 ins_encode %{
8655 Register Rdst = $dst$$Register;
8656 Register Rsrc = $src$$Register;
8657 Label skip;
8658 __ bsrq(Rdst, Rsrc);
8659 __ jccb(Assembler::notZero, skip);
8660 __ movl(Rdst, -1);
8661 __ bind(skip);
8662 __ negl(Rdst);
8663 __ addl(Rdst, BitsPerLong - 1);
8664 %}
8665 ins_pipe(ialu_reg);
8666 %}
8667
8668 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8669 predicate(UseCountTrailingZerosInstruction);
8670 match(Set dst (CountTrailingZerosI src));
8671 effect(KILL cr);
8672
8673 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8674 ins_encode %{
8675 __ tzcntl($dst$$Register, $src$$Register);
8676 %}
8677 ins_pipe(ialu_reg);
8678 %}
8679
8680 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8681 predicate(UseCountTrailingZerosInstruction);
8682 match(Set dst (CountTrailingZerosI (LoadI src)));
8683 effect(KILL cr);
8684 ins_cost(175);
8685 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8686 ins_encode %{
8687 __ tzcntl($dst$$Register, $src$$Address);
8688 %}
8689 ins_pipe(ialu_reg_mem);
8690 %}
8691
8692 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
8693 predicate(!UseCountTrailingZerosInstruction);
8694 match(Set dst (CountTrailingZerosI src));
8695 effect(KILL cr);
8696
8697 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
8698 "jnz done\n\t"
8699 "movl $dst, 32\n"
8700 "done:" %}
8701 ins_encode %{
8702 Register Rdst = $dst$$Register;
8703 Label done;
8704 __ bsfl(Rdst, $src$$Register);
8705 __ jccb(Assembler::notZero, done);
8706 __ movl(Rdst, BitsPerInt);
8707 __ bind(done);
8708 %}
8709 ins_pipe(ialu_reg);
8710 %}
8711
8712 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8713 predicate(UseCountTrailingZerosInstruction);
8714 match(Set dst (CountTrailingZerosL src));
8715 effect(KILL cr);
8716
8717 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8718 ins_encode %{
8719 __ tzcntq($dst$$Register, $src$$Register);
8720 %}
8721 ins_pipe(ialu_reg);
8722 %}
8723
8724 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8725 predicate(UseCountTrailingZerosInstruction);
8726 match(Set dst (CountTrailingZerosL (LoadL src)));
8727 effect(KILL cr);
8728 ins_cost(175);
8729 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8730 ins_encode %{
8731 __ tzcntq($dst$$Register, $src$$Address);
8732 %}
8733 ins_pipe(ialu_reg_mem);
8734 %}
8735
8736 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
8737 predicate(!UseCountTrailingZerosInstruction);
8738 match(Set dst (CountTrailingZerosL src));
8739 effect(KILL cr);
8740
8741 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
8742 "jnz done\n\t"
8743 "movl $dst, 64\n"
8744 "done:" %}
8745 ins_encode %{
8746 Register Rdst = $dst$$Register;
8747 Label done;
8748 __ bsfq(Rdst, $src$$Register);
8749 __ jccb(Assembler::notZero, done);
8750 __ movl(Rdst, BitsPerLong);
8751 __ bind(done);
8752 %}
8753 ins_pipe(ialu_reg);
8754 %}
8755
8756 //--------------- Reverse Operation Instructions ----------------
8757 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
8758 predicate(!VM_Version::supports_gfni());
8759 match(Set dst (ReverseI src));
8760 effect(TEMP dst, TEMP rtmp, KILL cr);
8761 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
8762 ins_encode %{
8763 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
8764 %}
8765 ins_pipe( ialu_reg );
8766 %}
8767
8768 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
8769 predicate(VM_Version::supports_gfni());
8770 match(Set dst (ReverseI src));
8771 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8772 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8773 ins_encode %{
8774 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
8775 %}
8776 ins_pipe( ialu_reg );
8777 %}
8778
8779 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
8780 predicate(!VM_Version::supports_gfni());
8781 match(Set dst (ReverseL src));
8782 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
8783 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
8784 ins_encode %{
8785 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
8786 %}
8787 ins_pipe( ialu_reg );
8788 %}
8789
8790 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
8791 predicate(VM_Version::supports_gfni());
8792 match(Set dst (ReverseL src));
8793 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8794 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8795 ins_encode %{
8796 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
8797 %}
8798 ins_pipe( ialu_reg );
8799 %}
8800
8801 //---------- Population Count Instructions -------------------------------------
8802
8803 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
8804 predicate(UsePopCountInstruction);
8805 match(Set dst (PopCountI src));
8806 effect(KILL cr);
8807
8808 format %{ "popcnt $dst, $src" %}
8809 ins_encode %{
8810 __ popcntl($dst$$Register, $src$$Register);
8811 %}
8812 ins_pipe(ialu_reg);
8813 %}
8814
8815 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8816 predicate(UsePopCountInstruction);
8817 match(Set dst (PopCountI (LoadI mem)));
8818 effect(KILL cr);
8819
8820 format %{ "popcnt $dst, $mem" %}
8821 ins_encode %{
8822 __ popcntl($dst$$Register, $mem$$Address);
8823 %}
8824 ins_pipe(ialu_reg);
8825 %}
8826
8827 // Note: Long.bitCount(long) returns an int.
8828 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
8829 predicate(UsePopCountInstruction);
8830 match(Set dst (PopCountL src));
8831 effect(KILL cr);
8832
8833 format %{ "popcnt $dst, $src" %}
8834 ins_encode %{
8835 __ popcntq($dst$$Register, $src$$Register);
8836 %}
8837 ins_pipe(ialu_reg);
8838 %}
8839
8840 // Note: Long.bitCount(long) returns an int.
8841 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8842 predicate(UsePopCountInstruction);
8843 match(Set dst (PopCountL (LoadL mem)));
8844 effect(KILL cr);
8845
8846 format %{ "popcnt $dst, $mem" %}
8847 ins_encode %{
8848 __ popcntq($dst$$Register, $mem$$Address);
8849 %}
8850 ins_pipe(ialu_reg);
8851 %}
8852
8853
8854 //----------MemBar Instructions-----------------------------------------------
8855 // Memory barrier flavors
8856
8857 instruct membar_acquire()
8858 %{
8859 match(MemBarAcquire);
8860 match(LoadFence);
8861 ins_cost(0);
8862
8863 size(0);
8864 format %{ "MEMBAR-acquire ! (empty encoding)" %}
8865 ins_encode();
8866 ins_pipe(empty);
8867 %}
8868
8869 instruct membar_acquire_lock()
8870 %{
8871 match(MemBarAcquireLock);
8872 ins_cost(0);
8873
8874 size(0);
8875 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
8876 ins_encode();
8877 ins_pipe(empty);
8878 %}
8879
8880 instruct membar_release()
8881 %{
8882 match(MemBarRelease);
8883 match(StoreFence);
8884 ins_cost(0);
8885
8886 size(0);
8887 format %{ "MEMBAR-release ! (empty encoding)" %}
8888 ins_encode();
8889 ins_pipe(empty);
8890 %}
8891
8892 instruct membar_release_lock()
8893 %{
8894 match(MemBarReleaseLock);
8895 ins_cost(0);
8896
8897 size(0);
8898 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
8899 ins_encode();
8900 ins_pipe(empty);
8901 %}
8902
8903 instruct membar_storeload(rFlagsReg cr) %{
8904 match(MemBarStoreLoad);
8905 effect(KILL cr);
8906 ins_cost(400);
8907
8908 format %{
8909 $$template
8910 $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
8911 %}
8912 ins_encode %{
8913 __ membar(Assembler::StoreLoad);
8914 %}
8915 ins_pipe(pipe_slow);
8916 %}
8917
8918 instruct membar_volatile(rFlagsReg cr) %{
8919 match(MemBarVolatile);
8920 effect(KILL cr);
8921 ins_cost(400);
8922
8923 format %{
8924 $$template
8925 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
8926 %}
8927 ins_encode %{
8928 __ membar(Assembler::StoreLoad);
8929 %}
8930 ins_pipe(pipe_slow);
8931 %}
8932
8933 instruct unnecessary_membar_volatile()
8934 %{
8935 match(MemBarVolatile);
8936 predicate(Matcher::post_store_load_barrier(n));
8937 ins_cost(0);
8938
8939 size(0);
8940 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
8941 ins_encode();
8942 ins_pipe(empty);
8943 %}
8944
8945 instruct membar_full(rFlagsReg cr) %{
8946 match(MemBarFull);
8947 effect(KILL cr);
8948 ins_cost(400);
8949
8950 format %{
8951 $$template
8952 $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
8953 %}
8954 ins_encode %{
8955 __ membar(Assembler::StoreLoad);
8956 %}
8957 ins_pipe(pipe_slow);
8958 %}
8959
8960 instruct membar_storestore() %{
8961 match(MemBarStoreStore);
8962 match(StoreStoreFence);
8963 ins_cost(0);
8964
8965 size(0);
8966 format %{ "MEMBAR-storestore (empty encoding)" %}
8967 ins_encode( );
8968 ins_pipe(empty);
8969 %}
8970
8971 //----------Move Instructions--------------------------------------------------
8972
8973 instruct castX2P(rRegP dst, rRegL src)
8974 %{
8975 match(Set dst (CastX2P src));
8976
8977 format %{ "movq $dst, $src\t# long->ptr" %}
8978 ins_encode %{
8979 if ($dst$$reg != $src$$reg) {
8980 __ movptr($dst$$Register, $src$$Register);
8981 }
8982 %}
8983 ins_pipe(ialu_reg_reg); // XXX
8984 %}
8985
8986 instruct castI2N(rRegN dst, rRegI src)
8987 %{
8988 match(Set dst (CastI2N src));
8989
8990 format %{ "movq $dst, $src\t# int -> narrow ptr" %}
8991 ins_encode %{
8992 if ($dst$$reg != $src$$reg) {
8993 __ movl($dst$$Register, $src$$Register);
8994 }
8995 %}
8996 ins_pipe(ialu_reg_reg); // XXX
8997 %}
8998
8999 instruct castN2X(rRegL dst, rRegN src)
9000 %{
9001 match(Set dst (CastP2X src));
9002
9003 format %{ "movq $dst, $src\t# ptr -> long" %}
9004 ins_encode %{
9005 if ($dst$$reg != $src$$reg) {
9006 __ movptr($dst$$Register, $src$$Register);
9007 }
9008 %}
9009 ins_pipe(ialu_reg_reg); // XXX
9010 %}
9011
9012 instruct castP2X(rRegL dst, rRegP src)
9013 %{
9014 match(Set dst (CastP2X src));
9015
9016 format %{ "movq $dst, $src\t# ptr -> long" %}
9017 ins_encode %{
9018 if ($dst$$reg != $src$$reg) {
9019 __ movptr($dst$$Register, $src$$Register);
9020 }
9021 %}
9022 ins_pipe(ialu_reg_reg); // XXX
9023 %}
9024
9025 // Convert oop into int for vectors alignment masking
9026 instruct convP2I(rRegI dst, rRegP src)
9027 %{
9028 match(Set dst (ConvL2I (CastP2X src)));
9029
9030 format %{ "movl $dst, $src\t# ptr -> int" %}
9031 ins_encode %{
9032 __ movl($dst$$Register, $src$$Register);
9033 %}
9034 ins_pipe(ialu_reg_reg); // XXX
9035 %}
9036
9037 // Convert compressed oop into int for vectors alignment masking
9038 // in case of 32bit oops (heap < 4Gb).
9039 instruct convN2I(rRegI dst, rRegN src)
9040 %{
9041 predicate(CompressedOops::shift() == 0);
9042 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
9043
9044 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
9045 ins_encode %{
9046 __ movl($dst$$Register, $src$$Register);
9047 %}
9048 ins_pipe(ialu_reg_reg); // XXX
9049 %}
9050
9051 // Convert oop pointer into compressed form
9052 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
9053 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9054 match(Set dst (EncodeP src));
9055 effect(KILL cr);
9056 format %{ "encode_heap_oop $dst,$src" %}
9057 ins_encode %{
9058 Register s = $src$$Register;
9059 Register d = $dst$$Register;
9060 if (s != d) {
9061 __ movq(d, s);
9062 }
9063 __ encode_heap_oop(d);
9064 %}
9065 ins_pipe(ialu_reg_long);
9066 %}
9067
9068 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9069 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9070 match(Set dst (EncodeP src));
9071 effect(KILL cr);
9072 format %{ "encode_heap_oop_not_null $dst,$src" %}
9073 ins_encode %{
9074 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
9075 %}
9076 ins_pipe(ialu_reg_long);
9077 %}
9078
9079 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
9080 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
9081 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
9082 match(Set dst (DecodeN src));
9083 effect(KILL cr);
9084 format %{ "decode_heap_oop $dst,$src" %}
9085 ins_encode %{
9086 Register s = $src$$Register;
9087 Register d = $dst$$Register;
9088 if (s != d) {
9089 __ movq(d, s);
9090 }
9091 __ decode_heap_oop(d);
9092 %}
9093 ins_pipe(ialu_reg_long);
9094 %}
9095
9096 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9097 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9098 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9099 match(Set dst (DecodeN src));
9100 effect(KILL cr);
9101 format %{ "decode_heap_oop_not_null $dst,$src" %}
9102 ins_encode %{
9103 Register s = $src$$Register;
9104 Register d = $dst$$Register;
9105 if (s != d) {
9106 __ decode_heap_oop_not_null(d, s);
9107 } else {
9108 __ decode_heap_oop_not_null(d);
9109 }
9110 %}
9111 ins_pipe(ialu_reg_long);
9112 %}
9113
9114 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9115 match(Set dst (EncodePKlass src));
9116 effect(TEMP dst, KILL cr);
9117 format %{ "encode_and_move_klass_not_null $dst,$src" %}
9118 ins_encode %{
9119 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
9120 %}
9121 ins_pipe(ialu_reg_long);
9122 %}
9123
9124 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9125 match(Set dst (DecodeNKlass src));
9126 effect(TEMP dst, KILL cr);
9127 format %{ "decode_and_move_klass_not_null $dst,$src" %}
9128 ins_encode %{
9129 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
9130 %}
9131 ins_pipe(ialu_reg_long);
9132 %}
9133
9134 //----------Conditional Move---------------------------------------------------
9135 // Jump
9136 // dummy instruction for generating temp registers
9137 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
9138 match(Jump (LShiftL switch_val shift));
9139 ins_cost(350);
9140 predicate(false);
9141 effect(TEMP dest);
9142
9143 format %{ "leaq $dest, [$constantaddress]\n\t"
9144 "jmp [$dest + $switch_val << $shift]\n\t" %}
9145 ins_encode %{
9146 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9147 // to do that and the compiler is using that register as one it can allocate.
9148 // So we build it all by hand.
9149 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
9150 // ArrayAddress dispatch(table, index);
9151 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
9152 __ lea($dest$$Register, $constantaddress);
9153 __ jmp(dispatch);
9154 %}
9155 ins_pipe(pipe_jmp);
9156 %}
9157
9158 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
9159 match(Jump (AddL (LShiftL switch_val shift) offset));
9160 ins_cost(350);
9161 effect(TEMP dest);
9162
9163 format %{ "leaq $dest, [$constantaddress]\n\t"
9164 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
9165 ins_encode %{
9166 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9167 // to do that and the compiler is using that register as one it can allocate.
9168 // So we build it all by hand.
9169 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9170 // ArrayAddress dispatch(table, index);
9171 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9172 __ lea($dest$$Register, $constantaddress);
9173 __ jmp(dispatch);
9174 %}
9175 ins_pipe(pipe_jmp);
9176 %}
9177
9178 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
9179 match(Jump switch_val);
9180 ins_cost(350);
9181 effect(TEMP dest);
9182
9183 format %{ "leaq $dest, [$constantaddress]\n\t"
9184 "jmp [$dest + $switch_val]\n\t" %}
9185 ins_encode %{
9186 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9187 // to do that and the compiler is using that register as one it can allocate.
9188 // So we build it all by hand.
9189 // Address index(noreg, switch_reg, Address::times_1);
9190 // ArrayAddress dispatch(table, index);
9191 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
9192 __ lea($dest$$Register, $constantaddress);
9193 __ jmp(dispatch);
9194 %}
9195 ins_pipe(pipe_jmp);
9196 %}
9197
9198 // Conditional move
9199 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
9200 %{
9201 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9202 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9203
9204 ins_cost(100); // XXX
9205 format %{ "setbn$cop $dst\t# signed, int" %}
9206 ins_encode %{
9207 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9208 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9209 %}
9210 ins_pipe(ialu_reg);
9211 %}
9212
9213 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
9214 %{
9215 predicate(!UseAPX);
9216 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9217
9218 ins_cost(200); // XXX
9219 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9220 ins_encode %{
9221 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9222 %}
9223 ins_pipe(pipe_cmov_reg);
9224 %}
9225
9226 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
9227 %{
9228 predicate(UseAPX);
9229 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9230
9231 ins_cost(200);
9232 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
9233 ins_encode %{
9234 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9235 %}
9236 ins_pipe(pipe_cmov_reg);
9237 %}
9238
9239 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
9240 %{
9241 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9242 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9243
9244 ins_cost(100); // XXX
9245 format %{ "setbn$cop $dst\t# unsigned, int" %}
9246 ins_encode %{
9247 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9248 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9249 %}
9250 ins_pipe(ialu_reg);
9251 %}
9252
9253 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
9254 predicate(!UseAPX);
9255 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9256
9257 ins_cost(200); // XXX
9258 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9259 ins_encode %{
9260 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9261 %}
9262 ins_pipe(pipe_cmov_reg);
9263 %}
9264
9265 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
9266 predicate(UseAPX);
9267 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9268
9269 ins_cost(200);
9270 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
9271 ins_encode %{
9272 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9273 %}
9274 ins_pipe(pipe_cmov_reg);
9275 %}
9276
9277 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9278 %{
9279 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9280 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9281
9282 ins_cost(100); // XXX
9283 format %{ "setbn$cop $dst\t# unsigned, int" %}
9284 ins_encode %{
9285 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9286 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9287 %}
9288 ins_pipe(ialu_reg);
9289 %}
9290
9291 instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9292 %{
9293 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9294 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9295
9296 ins_cost(100); // XXX
9297 format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
9298 ins_encode %{
9299 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9300 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9301 %}
9302 ins_pipe(ialu_reg);
9303 %}
9304
9305 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9306 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9307
9308 ins_cost(200);
9309 expand %{
9310 cmovI_regU(cop, cr, dst, src);
9311 %}
9312 %}
9313
9314 instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
9315 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9316
9317 ins_cost(200);
9318 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
9319 ins_encode %{
9320 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9321 %}
9322 ins_pipe(pipe_cmov_reg);
9323 %}
9324
9325 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9326 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9327 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9328
9329 ins_cost(200); // XXX
9330 format %{ "cmovpl $dst, $src\n\t"
9331 "cmovnel $dst, $src" %}
9332 ins_encode %{
9333 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9334 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9335 %}
9336 ins_pipe(pipe_cmov_reg);
9337 %}
9338
9339 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9340 // inputs of the CMove
9341 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9342 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9343 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9344 effect(TEMP dst);
9345
9346 ins_cost(200); // XXX
9347 format %{ "cmovpl $dst, $src\n\t"
9348 "cmovnel $dst, $src" %}
9349 ins_encode %{
9350 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9351 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9352 %}
9353 ins_pipe(pipe_cmov_reg);
9354 %}
9355
9356 // Conditional move
9357 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
9358 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9359
9360 ins_cost(250); // XXX
9361 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9362 ins_encode %{
9363 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9364 %}
9365 ins_pipe(pipe_cmov_mem);
9366 %}
9367
9368 // Conditional move
9369 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
9370 %{
9371 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9372
9373 ins_cost(250); // XXX
9374 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9375 ins_encode %{
9376 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9377 %}
9378 ins_pipe(pipe_cmov_mem);
9379 %}
9380
9381 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
9382 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9383
9384 ins_cost(250);
9385 expand %{
9386 cmovI_memU(cop, cr, dst, src);
9387 %}
9388 %}
9389
9390 instruct cmovI_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI dst, memory src) %{
9391 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9392
9393 ins_cost(250); // XXX
9394 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9395 ins_encode %{
9396 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9397 %}
9398 ins_pipe(pipe_cmov_mem);
9399 %}
9400
9401 // Conditional move
9402 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
9403 %{
9404 predicate(!UseAPX);
9405 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9406
9407 ins_cost(200); // XXX
9408 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
9409 ins_encode %{
9410 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9411 %}
9412 ins_pipe(pipe_cmov_reg);
9413 %}
9414
9415 // Conditional move ndd
9416 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
9417 %{
9418 predicate(UseAPX);
9419 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9420
9421 ins_cost(200);
9422 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
9423 ins_encode %{
9424 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9425 %}
9426 ins_pipe(pipe_cmov_reg);
9427 %}
9428
9429 // Conditional move
9430 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
9431 %{
9432 predicate(!UseAPX);
9433 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9434
9435 ins_cost(200); // XXX
9436 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
9437 ins_encode %{
9438 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9439 %}
9440 ins_pipe(pipe_cmov_reg);
9441 %}
9442
9443 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9444 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9445
9446 ins_cost(200);
9447 expand %{
9448 cmovN_regU(cop, cr, dst, src);
9449 %}
9450 %}
9451
9452 // Conditional move ndd
9453 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
9454 %{
9455 predicate(UseAPX);
9456 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9457
9458 ins_cost(200);
9459 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
9460 ins_encode %{
9461 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9462 %}
9463 ins_pipe(pipe_cmov_reg);
9464 %}
9465
9466 instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
9467 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9468
9469 ins_cost(200);
9470 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
9471 ins_encode %{
9472 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9473 %}
9474 ins_pipe(pipe_cmov_reg);
9475 %}
9476
9477 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9478 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9479 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9480
9481 ins_cost(200); // XXX
9482 format %{ "cmovpl $dst, $src\n\t"
9483 "cmovnel $dst, $src" %}
9484 ins_encode %{
9485 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9486 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9487 %}
9488 ins_pipe(pipe_cmov_reg);
9489 %}
9490
9491 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9492 // inputs of the CMove
9493 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9494 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9495 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
9496
9497 ins_cost(200); // XXX
9498 format %{ "cmovpl $dst, $src\n\t"
9499 "cmovnel $dst, $src" %}
9500 ins_encode %{
9501 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9502 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9503 %}
9504 ins_pipe(pipe_cmov_reg);
9505 %}
9506
9507 // Conditional move
9508 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
9509 %{
9510 predicate(!UseAPX);
9511 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9512
9513 ins_cost(200); // XXX
9514 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
9515 ins_encode %{
9516 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9517 %}
9518 ins_pipe(pipe_cmov_reg); // XXX
9519 %}
9520
9521 // Conditional move ndd
9522 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
9523 %{
9524 predicate(UseAPX);
9525 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9526
9527 ins_cost(200);
9528 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
9529 ins_encode %{
9530 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9531 %}
9532 ins_pipe(pipe_cmov_reg);
9533 %}
9534
9535 // Conditional move
9536 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
9537 %{
9538 predicate(!UseAPX);
9539 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9540
9541 ins_cost(200); // XXX
9542 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
9543 ins_encode %{
9544 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9545 %}
9546 ins_pipe(pipe_cmov_reg); // XXX
9547 %}
9548
9549 // Conditional move ndd
9550 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
9551 %{
9552 predicate(UseAPX);
9553 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9554
9555 ins_cost(200);
9556 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
9557 ins_encode %{
9558 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9559 %}
9560 ins_pipe(pipe_cmov_reg);
9561 %}
9562
9563 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9564 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9565
9566 ins_cost(200);
9567 expand %{
9568 cmovP_regU(cop, cr, dst, src);
9569 %}
9570 %}
9571
9572 instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
9573 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9574
9575 ins_cost(200);
9576 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
9577 ins_encode %{
9578 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9579 %}
9580 ins_pipe(pipe_cmov_reg);
9581 %}
9582
9583 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9584 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9585 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9586
9587 ins_cost(200); // XXX
9588 format %{ "cmovpq $dst, $src\n\t"
9589 "cmovneq $dst, $src" %}
9590 ins_encode %{
9591 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9592 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9593 %}
9594 ins_pipe(pipe_cmov_reg);
9595 %}
9596
9597 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9598 // inputs of the CMove
9599 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9600 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9601 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
9602
9603 ins_cost(200); // XXX
9604 format %{ "cmovpq $dst, $src\n\t"
9605 "cmovneq $dst, $src" %}
9606 ins_encode %{
9607 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9608 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9609 %}
9610 ins_pipe(pipe_cmov_reg);
9611 %}
9612
9613 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
9614 %{
9615 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9616 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9617
9618 ins_cost(100); // XXX
9619 format %{ "setbn$cop $dst\t# signed, long" %}
9620 ins_encode %{
9621 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9622 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9623 %}
9624 ins_pipe(ialu_reg);
9625 %}
9626
9627 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
9628 %{
9629 predicate(!UseAPX);
9630 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9631
9632 ins_cost(200); // XXX
9633 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9634 ins_encode %{
9635 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9636 %}
9637 ins_pipe(pipe_cmov_reg); // XXX
9638 %}
9639
9640 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
9641 %{
9642 predicate(UseAPX);
9643 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9644
9645 ins_cost(200);
9646 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
9647 ins_encode %{
9648 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9649 %}
9650 ins_pipe(pipe_cmov_reg);
9651 %}
9652
9653 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
9654 %{
9655 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9656
9657 ins_cost(200); // XXX
9658 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9659 ins_encode %{
9660 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9661 %}
9662 ins_pipe(pipe_cmov_mem); // XXX
9663 %}
9664
9665 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
9666 %{
9667 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9668 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9669
9670 ins_cost(100); // XXX
9671 format %{ "setbn$cop $dst\t# unsigned, long" %}
9672 ins_encode %{
9673 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9674 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9675 %}
9676 ins_pipe(ialu_reg);
9677 %}
9678
9679 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
9680 %{
9681 predicate(!UseAPX);
9682 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9683
9684 ins_cost(200); // XXX
9685 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9686 ins_encode %{
9687 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9688 %}
9689 ins_pipe(pipe_cmov_reg); // XXX
9690 %}
9691
9692 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
9693 %{
9694 predicate(UseAPX);
9695 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9696
9697 ins_cost(200);
9698 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
9699 ins_encode %{
9700 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9701 %}
9702 ins_pipe(pipe_cmov_reg);
9703 %}
9704
9705 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9706 %{
9707 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9708 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9709
9710 ins_cost(100); // XXX
9711 format %{ "setbn$cop $dst\t# unsigned, long" %}
9712 ins_encode %{
9713 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9714 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9715 %}
9716 ins_pipe(ialu_reg);
9717 %}
9718
9719 instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9720 %{
9721 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9722 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9723
9724 ins_cost(100); // XXX
9725 format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
9726 ins_encode %{
9727 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9728 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9729 %}
9730 ins_pipe(ialu_reg);
9731 %}
9732
9733 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9734 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9735
9736 ins_cost(200);
9737 expand %{
9738 cmovL_regU(cop, cr, dst, src);
9739 %}
9740 %}
9741
9742 instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
9743 %{
9744 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9745
9746 ins_cost(200);
9747 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
9748 ins_encode %{
9749 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9750 %}
9751 ins_pipe(pipe_cmov_reg);
9752 %}
9753
9754 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9755 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9756 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9757
9758 ins_cost(200); // XXX
9759 format %{ "cmovpq $dst, $src\n\t"
9760 "cmovneq $dst, $src" %}
9761 ins_encode %{
9762 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9763 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9764 %}
9765 ins_pipe(pipe_cmov_reg);
9766 %}
9767
9768 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9769 // inputs of the CMove
9770 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9771 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9772 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9773
9774 ins_cost(200); // XXX
9775 format %{ "cmovpq $dst, $src\n\t"
9776 "cmovneq $dst, $src" %}
9777 ins_encode %{
9778 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9779 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9780 %}
9781 ins_pipe(pipe_cmov_reg);
9782 %}
9783
9784 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
9785 %{
9786 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9787
9788 ins_cost(200); // XXX
9789 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9790 ins_encode %{
9791 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9792 %}
9793 ins_pipe(pipe_cmov_mem); // XXX
9794 %}
9795
9796 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
9797 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9798
9799 ins_cost(200);
9800 expand %{
9801 cmovL_memU(cop, cr, dst, src);
9802 %}
9803 %}
9804
9805 instruct cmovL_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL dst, memory src) %{
9806 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9807
9808 ins_cost(200); // XXX
9809 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9810 ins_encode %{
9811 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9812 %}
9813 ins_pipe(pipe_cmov_mem); // XXX
9814 %}
9815
9816 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
9817 %{
9818 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9819
9820 ins_cost(200); // XXX
9821 format %{ "jn$cop skip\t# signed cmove float\n\t"
9822 "movss $dst, $src\n"
9823 "skip:" %}
9824 ins_encode %{
9825 Label Lskip;
9826 // Invert sense of branch from sense of CMOV
9827 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9828 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9829 __ bind(Lskip);
9830 %}
9831 ins_pipe(pipe_slow);
9832 %}
9833
9834 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
9835 %{
9836 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9837
9838 ins_cost(200); // XXX
9839 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
9840 "movss $dst, $src\n"
9841 "skip:" %}
9842 ins_encode %{
9843 Label Lskip;
9844 // Invert sense of branch from sense of CMOV
9845 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9846 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9847 __ bind(Lskip);
9848 %}
9849 ins_pipe(pipe_slow);
9850 %}
9851
9852 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
9853 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9854
9855 ins_cost(200);
9856 expand %{
9857 cmovF_regU(cop, cr, dst, src);
9858 %}
9859 %}
9860
9861 instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
9862 %{
9863 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9864
9865 ins_cost(200); // XXX
9866 format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
9867 "movss $dst, $src\n"
9868 "skip:" %}
9869 ins_encode %{
9870 Label Lskip;
9871 // Invert sense of branch from sense of CMOV
9872 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9873 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9874 __ bind(Lskip);
9875 %}
9876 ins_pipe(pipe_slow);
9877 %}
9878
9879 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
9880 %{
9881 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9882
9883 ins_cost(200); // XXX
9884 format %{ "jn$cop skip\t# signed cmove double\n\t"
9885 "movsd $dst, $src\n"
9886 "skip:" %}
9887 ins_encode %{
9888 Label Lskip;
9889 // Invert sense of branch from sense of CMOV
9890 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9891 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9892 __ bind(Lskip);
9893 %}
9894 ins_pipe(pipe_slow);
9895 %}
9896
9897 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
9898 %{
9899 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9900
9901 ins_cost(200); // XXX
9902 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
9903 "movsd $dst, $src\n"
9904 "skip:" %}
9905 ins_encode %{
9906 Label Lskip;
9907 // Invert sense of branch from sense of CMOV
9908 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9909 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9910 __ bind(Lskip);
9911 %}
9912 ins_pipe(pipe_slow);
9913 %}
9914
9915 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
9916 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9917
9918 ins_cost(200);
9919 expand %{
9920 cmovD_regU(cop, cr, dst, src);
9921 %}
9922 %}
9923
9924 instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
9925 %{
9926 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9927
9928 ins_cost(200); // XXX
9929 format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
9930 "movsd $dst, $src\n"
9931 "skip:" %}
9932 ins_encode %{
9933 Label Lskip;
9934 // Invert sense of branch from sense of CMOV
9935 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9936 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9937 __ bind(Lskip);
9938 %}
9939 ins_pipe(pipe_slow);
9940 %}
9941
9942 //----------Arithmetic Instructions--------------------------------------------
9943 //----------Addition Instructions----------------------------------------------
9944
9945 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9946 %{
9947 predicate(!UseAPX);
9948 match(Set dst (AddI dst src));
9949 effect(KILL cr);
9950 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9951 format %{ "addl $dst, $src\t# int" %}
9952 ins_encode %{
9953 __ addl($dst$$Register, $src$$Register);
9954 %}
9955 ins_pipe(ialu_reg_reg);
9956 %}
9957
9958 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
9959 %{
9960 predicate(UseAPX);
9961 match(Set dst (AddI src1 src2));
9962 effect(KILL cr);
9963 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
9964
9965 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9966 ins_encode %{
9967 __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
9968 %}
9969 ins_pipe(ialu_reg_reg);
9970 %}
9971
9972 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9973 %{
9974 predicate(!UseAPX);
9975 match(Set dst (AddI dst src));
9976 effect(KILL cr);
9977 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9978
9979 format %{ "addl $dst, $src\t# int" %}
9980 ins_encode %{
9981 __ addl($dst$$Register, $src$$constant);
9982 %}
9983 ins_pipe( ialu_reg );
9984 %}
9985
9986 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
9987 %{
9988 predicate(UseAPX);
9989 match(Set dst (AddI src1 src2));
9990 effect(KILL cr);
9991 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
9992
9993 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9994 ins_encode %{
9995 __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
9996 %}
9997 ins_pipe( ialu_reg );
9998 %}
9999
10000 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10001 %{
10002 match(Set dst (AddI dst (LoadI src)));
10003 effect(KILL cr);
10004 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10005
10006 ins_cost(150); // XXX
10007 format %{ "addl $dst, $src\t# int" %}
10008 ins_encode %{
10009 __ addl($dst$$Register, $src$$Address);
10010 %}
10011 ins_pipe(ialu_reg_mem);
10012 %}
10013
10014 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10015 %{
10016 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10017 effect(KILL cr);
10018 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10019
10020 ins_cost(150); // XXX
10021 format %{ "addl $dst, $src\t# int" %}
10022 ins_encode %{
10023 __ addl($dst$$Address, $src$$Register);
10024 %}
10025 ins_pipe(ialu_mem_reg);
10026 %}
10027
10028 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
10029 %{
10030 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10031 effect(KILL cr);
10032 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10033
10034
10035 ins_cost(125); // XXX
10036 format %{ "addl $dst, $src\t# int" %}
10037 ins_encode %{
10038 __ addl($dst$$Address, $src$$constant);
10039 %}
10040 ins_pipe(ialu_mem_imm);
10041 %}
10042
10043 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
10044 %{
10045 predicate(!UseAPX && UseIncDec);
10046 match(Set dst (AddI dst src));
10047 effect(KILL cr);
10048
10049 format %{ "incl $dst\t# int" %}
10050 ins_encode %{
10051 __ incrementl($dst$$Register);
10052 %}
10053 ins_pipe(ialu_reg);
10054 %}
10055
10056 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
10057 %{
10058 predicate(UseAPX && UseIncDec);
10059 match(Set dst (AddI src val));
10060 effect(KILL cr);
10061 flag(PD::Flag_ndd_demotable_opr1);
10062
10063 format %{ "eincl $dst, $src\t# int ndd" %}
10064 ins_encode %{
10065 __ eincl($dst$$Register, $src$$Register, false);
10066 %}
10067 ins_pipe(ialu_reg);
10068 %}
10069
10070 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
10071 %{
10072 predicate(UseIncDec);
10073 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10074 effect(KILL cr);
10075
10076 ins_cost(125); // XXX
10077 format %{ "incl $dst\t# int" %}
10078 ins_encode %{
10079 __ incrementl($dst$$Address);
10080 %}
10081 ins_pipe(ialu_mem_imm);
10082 %}
10083
10084 // XXX why does that use AddI
10085 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
10086 %{
10087 predicate(!UseAPX && UseIncDec);
10088 match(Set dst (AddI dst src));
10089 effect(KILL cr);
10090
10091 format %{ "decl $dst\t# int" %}
10092 ins_encode %{
10093 __ decrementl($dst$$Register);
10094 %}
10095 ins_pipe(ialu_reg);
10096 %}
10097
10098 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
10099 %{
10100 predicate(UseAPX && UseIncDec);
10101 match(Set dst (AddI src val));
10102 effect(KILL cr);
10103 flag(PD::Flag_ndd_demotable_opr1);
10104
10105 format %{ "edecl $dst, $src\t# int ndd" %}
10106 ins_encode %{
10107 __ edecl($dst$$Register, $src$$Register, false);
10108 %}
10109 ins_pipe(ialu_reg);
10110 %}
10111
10112 // XXX why does that use AddI
10113 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
10114 %{
10115 predicate(UseIncDec);
10116 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10117 effect(KILL cr);
10118
10119 ins_cost(125); // XXX
10120 format %{ "decl $dst\t# int" %}
10121 ins_encode %{
10122 __ decrementl($dst$$Address);
10123 %}
10124 ins_pipe(ialu_mem_imm);
10125 %}
10126
10127 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
10128 %{
10129 predicate(VM_Version::supports_fast_2op_lea());
10130 match(Set dst (AddI (LShiftI index scale) disp));
10131
10132 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
10133 ins_encode %{
10134 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10135 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10136 %}
10137 ins_pipe(ialu_reg_reg);
10138 %}
10139
10140 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
10141 %{
10142 predicate(VM_Version::supports_fast_3op_lea());
10143 match(Set dst (AddI (AddI base index) disp));
10144
10145 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
10146 ins_encode %{
10147 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10148 %}
10149 ins_pipe(ialu_reg_reg);
10150 %}
10151
10152 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
10153 %{
10154 predicate(VM_Version::supports_fast_2op_lea());
10155 match(Set dst (AddI base (LShiftI index scale)));
10156
10157 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
10158 ins_encode %{
10159 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10160 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
10161 %}
10162 ins_pipe(ialu_reg_reg);
10163 %}
10164
10165 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
10166 %{
10167 predicate(VM_Version::supports_fast_3op_lea());
10168 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
10169
10170 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
10171 ins_encode %{
10172 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10173 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10174 %}
10175 ins_pipe(ialu_reg_reg);
10176 %}
10177
10178 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10179 %{
10180 predicate(!UseAPX);
10181 match(Set dst (AddL dst src));
10182 effect(KILL cr);
10183 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10184
10185 format %{ "addq $dst, $src\t# long" %}
10186 ins_encode %{
10187 __ addq($dst$$Register, $src$$Register);
10188 %}
10189 ins_pipe(ialu_reg_reg);
10190 %}
10191
10192 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10193 %{
10194 predicate(UseAPX);
10195 match(Set dst (AddL src1 src2));
10196 effect(KILL cr);
10197 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
10198
10199 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10200 ins_encode %{
10201 __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
10202 %}
10203 ins_pipe(ialu_reg_reg);
10204 %}
10205
10206 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10207 %{
10208 predicate(!UseAPX);
10209 match(Set dst (AddL dst src));
10210 effect(KILL cr);
10211 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10212
10213 format %{ "addq $dst, $src\t# long" %}
10214 ins_encode %{
10215 __ addq($dst$$Register, $src$$constant);
10216 %}
10217 ins_pipe( ialu_reg );
10218 %}
10219
10220 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10221 %{
10222 predicate(UseAPX);
10223 match(Set dst (AddL src1 src2));
10224 effect(KILL cr);
10225 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10226
10227 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10228 ins_encode %{
10229 __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
10230 %}
10231 ins_pipe( ialu_reg );
10232 %}
10233
10234 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10235 %{
10236 match(Set dst (AddL dst (LoadL src)));
10237 effect(KILL cr);
10238 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10239
10240 ins_cost(150); // XXX
10241 format %{ "addq $dst, $src\t# long" %}
10242 ins_encode %{
10243 __ addq($dst$$Register, $src$$Address);
10244 %}
10245 ins_pipe(ialu_reg_mem);
10246 %}
10247
10248 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10249 %{
10250 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10251 effect(KILL cr);
10252 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10253
10254 ins_cost(150); // XXX
10255 format %{ "addq $dst, $src\t# long" %}
10256 ins_encode %{
10257 __ addq($dst$$Address, $src$$Register);
10258 %}
10259 ins_pipe(ialu_mem_reg);
10260 %}
10261
10262 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10263 %{
10264 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10265 effect(KILL cr);
10266 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10267
10268 ins_cost(125); // XXX
10269 format %{ "addq $dst, $src\t# long" %}
10270 ins_encode %{
10271 __ addq($dst$$Address, $src$$constant);
10272 %}
10273 ins_pipe(ialu_mem_imm);
10274 %}
10275
10276 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
10277 %{
10278 predicate(!UseAPX && UseIncDec);
10279 match(Set dst (AddL dst src));
10280 effect(KILL cr);
10281
10282 format %{ "incq $dst\t# long" %}
10283 ins_encode %{
10284 __ incrementq($dst$$Register);
10285 %}
10286 ins_pipe(ialu_reg);
10287 %}
10288
10289 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
10290 %{
10291 predicate(UseAPX && UseIncDec);
10292 match(Set dst (AddL src val));
10293 effect(KILL cr);
10294 flag(PD::Flag_ndd_demotable_opr1);
10295
10296 format %{ "eincq $dst, $src\t# long ndd" %}
10297 ins_encode %{
10298 __ eincq($dst$$Register, $src$$Register, false);
10299 %}
10300 ins_pipe(ialu_reg);
10301 %}
10302
10303 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
10304 %{
10305 predicate(UseIncDec);
10306 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10307 effect(KILL cr);
10308
10309 ins_cost(125); // XXX
10310 format %{ "incq $dst\t# long" %}
10311 ins_encode %{
10312 __ incrementq($dst$$Address);
10313 %}
10314 ins_pipe(ialu_mem_imm);
10315 %}
10316
10317 // XXX why does that use AddL
10318 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
10319 %{
10320 predicate(!UseAPX && UseIncDec);
10321 match(Set dst (AddL dst src));
10322 effect(KILL cr);
10323
10324 format %{ "decq $dst\t# long" %}
10325 ins_encode %{
10326 __ decrementq($dst$$Register);
10327 %}
10328 ins_pipe(ialu_reg);
10329 %}
10330
10331 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
10332 %{
10333 predicate(UseAPX && UseIncDec);
10334 match(Set dst (AddL src val));
10335 effect(KILL cr);
10336 flag(PD::Flag_ndd_demotable_opr1);
10337
10338 format %{ "edecq $dst, $src\t# long ndd" %}
10339 ins_encode %{
10340 __ edecq($dst$$Register, $src$$Register, false);
10341 %}
10342 ins_pipe(ialu_reg);
10343 %}
10344
10345 // XXX why does that use AddL
10346 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
10347 %{
10348 predicate(UseIncDec);
10349 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10350 effect(KILL cr);
10351
10352 ins_cost(125); // XXX
10353 format %{ "decq $dst\t# long" %}
10354 ins_encode %{
10355 __ decrementq($dst$$Address);
10356 %}
10357 ins_pipe(ialu_mem_imm);
10358 %}
10359
10360 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
10361 %{
10362 predicate(VM_Version::supports_fast_2op_lea());
10363 match(Set dst (AddL (LShiftL index scale) disp));
10364
10365 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
10366 ins_encode %{
10367 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10368 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10369 %}
10370 ins_pipe(ialu_reg_reg);
10371 %}
10372
10373 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
10374 %{
10375 predicate(VM_Version::supports_fast_3op_lea());
10376 match(Set dst (AddL (AddL base index) disp));
10377
10378 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
10379 ins_encode %{
10380 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10381 %}
10382 ins_pipe(ialu_reg_reg);
10383 %}
10384
10385 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
10386 %{
10387 predicate(VM_Version::supports_fast_2op_lea());
10388 match(Set dst (AddL base (LShiftL index scale)));
10389
10390 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
10391 ins_encode %{
10392 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10393 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
10394 %}
10395 ins_pipe(ialu_reg_reg);
10396 %}
10397
10398 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
10399 %{
10400 predicate(VM_Version::supports_fast_3op_lea());
10401 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
10402
10403 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
10404 ins_encode %{
10405 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10406 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10407 %}
10408 ins_pipe(ialu_reg_reg);
10409 %}
10410
10411 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
10412 %{
10413 match(Set dst (AddP dst src));
10414 effect(KILL cr);
10415 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10416
10417 format %{ "addq $dst, $src\t# ptr" %}
10418 ins_encode %{
10419 __ addq($dst$$Register, $src$$Register);
10420 %}
10421 ins_pipe(ialu_reg_reg);
10422 %}
10423
10424 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
10425 %{
10426 match(Set dst (AddP dst src));
10427 effect(KILL cr);
10428 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10429
10430 format %{ "addq $dst, $src\t# ptr" %}
10431 ins_encode %{
10432 __ addq($dst$$Register, $src$$constant);
10433 %}
10434 ins_pipe( ialu_reg );
10435 %}
10436
10437 // XXX addP mem ops ????
10438
10439 instruct checkCastPP(rRegP dst)
10440 %{
10441 match(Set dst (CheckCastPP dst));
10442
10443 size(0);
10444 format %{ "# checkcastPP of $dst" %}
10445 ins_encode(/* empty encoding */);
10446 ins_pipe(empty);
10447 %}
10448
10449 instruct castPP(rRegP dst)
10450 %{
10451 match(Set dst (CastPP dst));
10452
10453 size(0);
10454 format %{ "# castPP of $dst" %}
10455 ins_encode(/* empty encoding */);
10456 ins_pipe(empty);
10457 %}
10458
10459 instruct castII(rRegI dst)
10460 %{
10461 predicate(VerifyConstraintCasts == 0);
10462 match(Set dst (CastII dst));
10463
10464 size(0);
10465 format %{ "# castII of $dst" %}
10466 ins_encode(/* empty encoding */);
10467 ins_cost(0);
10468 ins_pipe(empty);
10469 %}
10470
10471 instruct castII_checked(rRegI dst, rFlagsReg cr)
10472 %{
10473 predicate(VerifyConstraintCasts > 0);
10474 match(Set dst (CastII dst));
10475
10476 effect(KILL cr);
10477 format %{ "# cast_checked_II $dst" %}
10478 ins_encode %{
10479 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
10480 %}
10481 ins_pipe(pipe_slow);
10482 %}
10483
10484 instruct castLL(rRegL dst)
10485 %{
10486 predicate(VerifyConstraintCasts == 0);
10487 match(Set dst (CastLL dst));
10488
10489 size(0);
10490 format %{ "# castLL of $dst" %}
10491 ins_encode(/* empty encoding */);
10492 ins_cost(0);
10493 ins_pipe(empty);
10494 %}
10495
10496 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
10497 %{
10498 predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
10499 match(Set dst (CastLL dst));
10500
10501 effect(KILL cr);
10502 format %{ "# cast_checked_LL $dst" %}
10503 ins_encode %{
10504 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
10505 %}
10506 ins_pipe(pipe_slow);
10507 %}
10508
10509 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
10510 %{
10511 predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
10512 match(Set dst (CastLL dst));
10513
10514 effect(KILL cr, TEMP tmp);
10515 format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
10516 ins_encode %{
10517 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
10518 %}
10519 ins_pipe(pipe_slow);
10520 %}
10521
10522 instruct castFF(regF dst)
10523 %{
10524 match(Set dst (CastFF dst));
10525
10526 size(0);
10527 format %{ "# castFF of $dst" %}
10528 ins_encode(/* empty encoding */);
10529 ins_cost(0);
10530 ins_pipe(empty);
10531 %}
10532
10533 instruct castHH(regF dst)
10534 %{
10535 match(Set dst (CastHH dst));
10536
10537 size(0);
10538 format %{ "# castHH of $dst" %}
10539 ins_encode(/* empty encoding */);
10540 ins_cost(0);
10541 ins_pipe(empty);
10542 %}
10543
10544 instruct castDD(regD dst)
10545 %{
10546 match(Set dst (CastDD dst));
10547
10548 size(0);
10549 format %{ "# castDD of $dst" %}
10550 ins_encode(/* empty encoding */);
10551 ins_cost(0);
10552 ins_pipe(empty);
10553 %}
10554
10555 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
10556 instruct compareAndSwapP(rRegI res,
10557 memory mem_ptr,
10558 rax_RegP oldval, rRegP newval,
10559 rFlagsReg cr)
10560 %{
10561 predicate(n->as_LoadStore()->barrier_data() == 0);
10562 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
10563 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
10564 effect(KILL cr, KILL oldval);
10565
10566 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10567 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10568 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10569 ins_encode %{
10570 __ lock();
10571 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10572 __ setcc(Assembler::equal, $res$$Register);
10573 %}
10574 ins_pipe( pipe_cmpxchg );
10575 %}
10576
10577 instruct compareAndSwapL(rRegI res,
10578 memory mem_ptr,
10579 rax_RegL oldval, rRegL newval,
10580 rFlagsReg cr)
10581 %{
10582 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
10583 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
10584 effect(KILL cr, KILL oldval);
10585
10586 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10587 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10588 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10589 ins_encode %{
10590 __ lock();
10591 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10592 __ setcc(Assembler::equal, $res$$Register);
10593 %}
10594 ins_pipe( pipe_cmpxchg );
10595 %}
10596
10597 instruct compareAndSwapI(rRegI res,
10598 memory mem_ptr,
10599 rax_RegI oldval, rRegI newval,
10600 rFlagsReg cr)
10601 %{
10602 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
10603 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
10604 effect(KILL cr, KILL oldval);
10605
10606 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10607 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10608 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10609 ins_encode %{
10610 __ lock();
10611 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10612 __ setcc(Assembler::equal, $res$$Register);
10613 %}
10614 ins_pipe( pipe_cmpxchg );
10615 %}
10616
10617 instruct compareAndSwapB(rRegI res,
10618 memory mem_ptr,
10619 rax_RegI oldval, rRegI newval,
10620 rFlagsReg cr)
10621 %{
10622 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
10623 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
10624 effect(KILL cr, KILL oldval);
10625
10626 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10627 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10628 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10629 ins_encode %{
10630 __ lock();
10631 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10632 __ setcc(Assembler::equal, $res$$Register);
10633 %}
10634 ins_pipe( pipe_cmpxchg );
10635 %}
10636
10637 instruct compareAndSwapS(rRegI res,
10638 memory mem_ptr,
10639 rax_RegI oldval, rRegI newval,
10640 rFlagsReg cr)
10641 %{
10642 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
10643 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
10644 effect(KILL cr, KILL oldval);
10645
10646 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10647 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10648 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10649 ins_encode %{
10650 __ lock();
10651 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10652 __ setcc(Assembler::equal, $res$$Register);
10653 %}
10654 ins_pipe( pipe_cmpxchg );
10655 %}
10656
10657 instruct compareAndSwapN(rRegI res,
10658 memory mem_ptr,
10659 rax_RegN oldval, rRegN newval,
10660 rFlagsReg cr) %{
10661 predicate(n->as_LoadStore()->barrier_data() == 0);
10662 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
10663 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
10664 effect(KILL cr, KILL oldval);
10665
10666 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10667 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10668 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10669 ins_encode %{
10670 __ lock();
10671 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10672 __ setcc(Assembler::equal, $res$$Register);
10673 %}
10674 ins_pipe( pipe_cmpxchg );
10675 %}
10676
10677 instruct compareAndExchangeB(
10678 memory mem_ptr,
10679 rax_RegI oldval, rRegI newval,
10680 rFlagsReg cr)
10681 %{
10682 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
10683 effect(KILL cr);
10684
10685 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10686 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10687 ins_encode %{
10688 __ lock();
10689 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10690 %}
10691 ins_pipe( pipe_cmpxchg );
10692 %}
10693
10694 instruct compareAndExchangeS(
10695 memory mem_ptr,
10696 rax_RegI oldval, rRegI newval,
10697 rFlagsReg cr)
10698 %{
10699 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
10700 effect(KILL cr);
10701
10702 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10703 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10704 ins_encode %{
10705 __ lock();
10706 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10707 %}
10708 ins_pipe( pipe_cmpxchg );
10709 %}
10710
10711 instruct compareAndExchangeI(
10712 memory mem_ptr,
10713 rax_RegI oldval, rRegI newval,
10714 rFlagsReg cr)
10715 %{
10716 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
10717 effect(KILL cr);
10718
10719 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10720 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10721 ins_encode %{
10722 __ lock();
10723 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10724 %}
10725 ins_pipe( pipe_cmpxchg );
10726 %}
10727
10728 instruct compareAndExchangeL(
10729 memory mem_ptr,
10730 rax_RegL oldval, rRegL newval,
10731 rFlagsReg cr)
10732 %{
10733 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
10734 effect(KILL cr);
10735
10736 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10737 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10738 ins_encode %{
10739 __ lock();
10740 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10741 %}
10742 ins_pipe( pipe_cmpxchg );
10743 %}
10744
10745 instruct compareAndExchangeN(
10746 memory mem_ptr,
10747 rax_RegN oldval, rRegN newval,
10748 rFlagsReg cr) %{
10749 predicate(n->as_LoadStore()->barrier_data() == 0);
10750 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
10751 effect(KILL cr);
10752
10753 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10754 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10755 ins_encode %{
10756 __ lock();
10757 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10758 %}
10759 ins_pipe( pipe_cmpxchg );
10760 %}
10761
10762 instruct compareAndExchangeP(
10763 memory mem_ptr,
10764 rax_RegP oldval, rRegP newval,
10765 rFlagsReg cr)
10766 %{
10767 predicate(n->as_LoadStore()->barrier_data() == 0);
10768 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
10769 effect(KILL cr);
10770
10771 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10772 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10773 ins_encode %{
10774 __ lock();
10775 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10776 %}
10777 ins_pipe( pipe_cmpxchg );
10778 %}
10779
10780 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10781 predicate(n->as_LoadStore()->result_not_used());
10782 match(Set dummy (GetAndAddB mem add));
10783 effect(KILL cr);
10784 format %{ "addb_lock $mem, $add" %}
10785 ins_encode %{
10786 __ lock();
10787 __ addb($mem$$Address, $add$$Register);
10788 %}
10789 ins_pipe(pipe_cmpxchg);
10790 %}
10791
10792 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10793 predicate(n->as_LoadStore()->result_not_used());
10794 match(Set dummy (GetAndAddB mem add));
10795 effect(KILL cr);
10796 format %{ "addb_lock $mem, $add" %}
10797 ins_encode %{
10798 __ lock();
10799 __ addb($mem$$Address, $add$$constant);
10800 %}
10801 ins_pipe(pipe_cmpxchg);
10802 %}
10803
10804 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
10805 predicate(!n->as_LoadStore()->result_not_used());
10806 match(Set newval (GetAndAddB mem newval));
10807 effect(KILL cr);
10808 format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
10809 ins_encode %{
10810 __ lock();
10811 __ xaddb($mem$$Address, $newval$$Register);
10812 __ narrow_subword_type($newval$$Register, T_BYTE);
10813 %}
10814 ins_pipe(pipe_cmpxchg);
10815 %}
10816
10817 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10818 predicate(n->as_LoadStore()->result_not_used());
10819 match(Set dummy (GetAndAddS mem add));
10820 effect(KILL cr);
10821 format %{ "addw_lock $mem, $add" %}
10822 ins_encode %{
10823 __ lock();
10824 __ addw($mem$$Address, $add$$Register);
10825 %}
10826 ins_pipe(pipe_cmpxchg);
10827 %}
10828
10829 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10830 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
10831 match(Set dummy (GetAndAddS mem add));
10832 effect(KILL cr);
10833 format %{ "addw_lock $mem, $add" %}
10834 ins_encode %{
10835 __ lock();
10836 __ addw($mem$$Address, $add$$constant);
10837 %}
10838 ins_pipe(pipe_cmpxchg);
10839 %}
10840
10841 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
10842 predicate(!n->as_LoadStore()->result_not_used());
10843 match(Set newval (GetAndAddS mem newval));
10844 effect(KILL cr);
10845 format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
10846 ins_encode %{
10847 __ lock();
10848 __ xaddw($mem$$Address, $newval$$Register);
10849 __ narrow_subword_type($newval$$Register, T_SHORT);
10850 %}
10851 ins_pipe(pipe_cmpxchg);
10852 %}
10853
10854 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10855 predicate(n->as_LoadStore()->result_not_used());
10856 match(Set dummy (GetAndAddI mem add));
10857 effect(KILL cr);
10858 format %{ "addl_lock $mem, $add" %}
10859 ins_encode %{
10860 __ lock();
10861 __ addl($mem$$Address, $add$$Register);
10862 %}
10863 ins_pipe(pipe_cmpxchg);
10864 %}
10865
10866 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10867 predicate(n->as_LoadStore()->result_not_used());
10868 match(Set dummy (GetAndAddI mem add));
10869 effect(KILL cr);
10870 format %{ "addl_lock $mem, $add" %}
10871 ins_encode %{
10872 __ lock();
10873 __ addl($mem$$Address, $add$$constant);
10874 %}
10875 ins_pipe(pipe_cmpxchg);
10876 %}
10877
10878 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
10879 predicate(!n->as_LoadStore()->result_not_used());
10880 match(Set newval (GetAndAddI mem newval));
10881 effect(KILL cr);
10882 format %{ "xaddl_lock $mem, $newval" %}
10883 ins_encode %{
10884 __ lock();
10885 __ xaddl($mem$$Address, $newval$$Register);
10886 %}
10887 ins_pipe(pipe_cmpxchg);
10888 %}
10889
10890 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
10891 predicate(n->as_LoadStore()->result_not_used());
10892 match(Set dummy (GetAndAddL mem add));
10893 effect(KILL cr);
10894 format %{ "addq_lock $mem, $add" %}
10895 ins_encode %{
10896 __ lock();
10897 __ addq($mem$$Address, $add$$Register);
10898 %}
10899 ins_pipe(pipe_cmpxchg);
10900 %}
10901
10902 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
10903 predicate(n->as_LoadStore()->result_not_used());
10904 match(Set dummy (GetAndAddL mem add));
10905 effect(KILL cr);
10906 format %{ "addq_lock $mem, $add" %}
10907 ins_encode %{
10908 __ lock();
10909 __ addq($mem$$Address, $add$$constant);
10910 %}
10911 ins_pipe(pipe_cmpxchg);
10912 %}
10913
10914 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
10915 predicate(!n->as_LoadStore()->result_not_used());
10916 match(Set newval (GetAndAddL mem newval));
10917 effect(KILL cr);
10918 format %{ "xaddq_lock $mem, $newval" %}
10919 ins_encode %{
10920 __ lock();
10921 __ xaddq($mem$$Address, $newval$$Register);
10922 %}
10923 ins_pipe(pipe_cmpxchg);
10924 %}
10925
10926 instruct xchgB( memory mem, rRegI newval) %{
10927 match(Set newval (GetAndSetB mem newval));
10928 format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
10929 ins_encode %{
10930 __ xchgb($newval$$Register, $mem$$Address);
10931 __ narrow_subword_type($newval$$Register, T_BYTE);
10932 %}
10933 ins_pipe( pipe_cmpxchg );
10934 %}
10935
10936 instruct xchgS( memory mem, rRegI newval) %{
10937 match(Set newval (GetAndSetS mem newval));
10938 format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
10939 ins_encode %{
10940 __ xchgw($newval$$Register, $mem$$Address);
10941 __ narrow_subword_type($newval$$Register, T_SHORT);
10942 %}
10943 ins_pipe( pipe_cmpxchg );
10944 %}
10945
10946 instruct xchgI( memory mem, rRegI newval) %{
10947 match(Set newval (GetAndSetI mem newval));
10948 format %{ "XCHGL $newval,[$mem]" %}
10949 ins_encode %{
10950 __ xchgl($newval$$Register, $mem$$Address);
10951 %}
10952 ins_pipe( pipe_cmpxchg );
10953 %}
10954
10955 instruct xchgL( memory mem, rRegL newval) %{
10956 match(Set newval (GetAndSetL mem newval));
10957 format %{ "XCHGL $newval,[$mem]" %}
10958 ins_encode %{
10959 __ xchgq($newval$$Register, $mem$$Address);
10960 %}
10961 ins_pipe( pipe_cmpxchg );
10962 %}
10963
10964 instruct xchgP( memory mem, rRegP newval) %{
10965 match(Set newval (GetAndSetP mem newval));
10966 predicate(n->as_LoadStore()->barrier_data() == 0);
10967 format %{ "XCHGQ $newval,[$mem]" %}
10968 ins_encode %{
10969 __ xchgq($newval$$Register, $mem$$Address);
10970 %}
10971 ins_pipe( pipe_cmpxchg );
10972 %}
10973
10974 instruct xchgN( memory mem, rRegN newval) %{
10975 predicate(n->as_LoadStore()->barrier_data() == 0);
10976 match(Set newval (GetAndSetN mem newval));
10977 format %{ "XCHGL $newval,$mem]" %}
10978 ins_encode %{
10979 __ xchgl($newval$$Register, $mem$$Address);
10980 %}
10981 ins_pipe( pipe_cmpxchg );
10982 %}
10983
10984 //----------Abs Instructions-------------------------------------------
10985
10986 // Integer Absolute Instructions
10987 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10988 %{
10989 match(Set dst (AbsI src));
10990 effect(TEMP dst, KILL cr);
10991 format %{ "xorl $dst, $dst\t# abs int\n\t"
10992 "subl $dst, $src\n\t"
10993 "cmovll $dst, $src" %}
10994 ins_encode %{
10995 __ xorl($dst$$Register, $dst$$Register);
10996 __ subl($dst$$Register, $src$$Register);
10997 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
10998 %}
10999
11000 ins_pipe(ialu_reg_reg);
11001 %}
11002
11003 // Long Absolute Instructions
11004 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11005 %{
11006 match(Set dst (AbsL src));
11007 effect(TEMP dst, KILL cr);
11008 format %{ "xorl $dst, $dst\t# abs long\n\t"
11009 "subq $dst, $src\n\t"
11010 "cmovlq $dst, $src" %}
11011 ins_encode %{
11012 __ xorl($dst$$Register, $dst$$Register);
11013 __ subq($dst$$Register, $src$$Register);
11014 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
11015 %}
11016
11017 ins_pipe(ialu_reg_reg);
11018 %}
11019
11020 //----------Subtraction Instructions-------------------------------------------
11021
11022 // Integer Subtraction Instructions
11023 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11024 %{
11025 predicate(!UseAPX);
11026 match(Set dst (SubI dst src));
11027 effect(KILL cr);
11028 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11029
11030 format %{ "subl $dst, $src\t# int" %}
11031 ins_encode %{
11032 __ subl($dst$$Register, $src$$Register);
11033 %}
11034 ins_pipe(ialu_reg_reg);
11035 %}
11036
11037 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11038 %{
11039 predicate(UseAPX);
11040 match(Set dst (SubI src1 src2));
11041 effect(KILL cr);
11042 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11043
11044 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
11045 ins_encode %{
11046 __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
11047 %}
11048 ins_pipe(ialu_reg_reg);
11049 %}
11050
11051 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
11052 %{
11053 predicate(UseAPX);
11054 match(Set dst (SubI src1 src2));
11055 effect(KILL cr);
11056 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11057
11058 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
11059 ins_encode %{
11060 __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
11061 %}
11062 ins_pipe(ialu_reg_reg);
11063 %}
11064
11065 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
11066 %{
11067 match(Set dst (SubI dst (LoadI src)));
11068 effect(KILL cr);
11069 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11070
11071 ins_cost(150);
11072 format %{ "subl $dst, $src\t# int" %}
11073 ins_encode %{
11074 __ subl($dst$$Register, $src$$Address);
11075 %}
11076 ins_pipe(ialu_reg_mem);
11077 %}
11078
11079 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
11080 %{
11081 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
11082 effect(KILL cr);
11083 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11084
11085 ins_cost(150);
11086 format %{ "subl $dst, $src\t# int" %}
11087 ins_encode %{
11088 __ subl($dst$$Address, $src$$Register);
11089 %}
11090 ins_pipe(ialu_mem_reg);
11091 %}
11092
11093 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11094 %{
11095 predicate(!UseAPX);
11096 match(Set dst (SubL dst src));
11097 effect(KILL cr);
11098 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11099
11100 format %{ "subq $dst, $src\t# long" %}
11101 ins_encode %{
11102 __ subq($dst$$Register, $src$$Register);
11103 %}
11104 ins_pipe(ialu_reg_reg);
11105 %}
11106
11107 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11108 %{
11109 predicate(UseAPX);
11110 match(Set dst (SubL src1 src2));
11111 effect(KILL cr);
11112 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11113
11114 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11115 ins_encode %{
11116 __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
11117 %}
11118 ins_pipe(ialu_reg_reg);
11119 %}
11120
11121 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11122 %{
11123 predicate(UseAPX);
11124 match(Set dst (SubL src1 src2));
11125 effect(KILL cr);
11126 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11127
11128 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11129 ins_encode %{
11130 __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
11131 %}
11132 ins_pipe(ialu_reg_reg);
11133 %}
11134
11135 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11136 %{
11137 match(Set dst (SubL dst (LoadL src)));
11138 effect(KILL cr);
11139 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11140
11141 ins_cost(150);
11142 format %{ "subq $dst, $src\t# long" %}
11143 ins_encode %{
11144 __ subq($dst$$Register, $src$$Address);
11145 %}
11146 ins_pipe(ialu_reg_mem);
11147 %}
11148
11149 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11150 %{
11151 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
11152 effect(KILL cr);
11153 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11154
11155 ins_cost(150);
11156 format %{ "subq $dst, $src\t# long" %}
11157 ins_encode %{
11158 __ subq($dst$$Address, $src$$Register);
11159 %}
11160 ins_pipe(ialu_mem_reg);
11161 %}
11162
11163 // Subtract from a pointer
11164 // XXX hmpf???
11165 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
11166 %{
11167 match(Set dst (AddP dst (SubI zero src)));
11168 effect(KILL cr);
11169
11170 format %{ "subq $dst, $src\t# ptr - int" %}
11171 ins_encode %{
11172 __ subq($dst$$Register, $src$$Register);
11173 %}
11174 ins_pipe(ialu_reg_reg);
11175 %}
11176
11177 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
11178 %{
11179 predicate(!UseAPX);
11180 match(Set dst (SubI zero dst));
11181 effect(KILL cr);
11182 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11183
11184 format %{ "negl $dst\t# int" %}
11185 ins_encode %{
11186 __ negl($dst$$Register);
11187 %}
11188 ins_pipe(ialu_reg);
11189 %}
11190
11191 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
11192 %{
11193 predicate(UseAPX);
11194 match(Set dst (SubI zero src));
11195 effect(KILL cr);
11196 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11197
11198 format %{ "enegl $dst, $src\t# int ndd" %}
11199 ins_encode %{
11200 __ enegl($dst$$Register, $src$$Register, false);
11201 %}
11202 ins_pipe(ialu_reg);
11203 %}
11204
11205 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
11206 %{
11207 predicate(!UseAPX);
11208 match(Set dst (NegI dst));
11209 effect(KILL cr);
11210 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11211
11212 format %{ "negl $dst\t# int" %}
11213 ins_encode %{
11214 __ negl($dst$$Register);
11215 %}
11216 ins_pipe(ialu_reg);
11217 %}
11218
11219 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
11220 %{
11221 predicate(UseAPX);
11222 match(Set dst (NegI src));
11223 effect(KILL cr);
11224 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11225
11226 format %{ "enegl $dst, $src\t# int ndd" %}
11227 ins_encode %{
11228 __ enegl($dst$$Register, $src$$Register, false);
11229 %}
11230 ins_pipe(ialu_reg);
11231 %}
11232
11233 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
11234 %{
11235 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
11236 effect(KILL cr);
11237 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11238
11239 format %{ "negl $dst\t# int" %}
11240 ins_encode %{
11241 __ negl($dst$$Address);
11242 %}
11243 ins_pipe(ialu_reg);
11244 %}
11245
11246 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
11247 %{
11248 predicate(!UseAPX);
11249 match(Set dst (SubL zero dst));
11250 effect(KILL cr);
11251 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11252
11253 format %{ "negq $dst\t# long" %}
11254 ins_encode %{
11255 __ negq($dst$$Register);
11256 %}
11257 ins_pipe(ialu_reg);
11258 %}
11259
11260 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
11261 %{
11262 predicate(UseAPX);
11263 match(Set dst (SubL zero src));
11264 effect(KILL cr);
11265 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11266
11267 format %{ "enegq $dst, $src\t# long ndd" %}
11268 ins_encode %{
11269 __ enegq($dst$$Register, $src$$Register, false);
11270 %}
11271 ins_pipe(ialu_reg);
11272 %}
11273
11274 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
11275 %{
11276 predicate(!UseAPX);
11277 match(Set dst (NegL dst));
11278 effect(KILL cr);
11279 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11280
11281 format %{ "negq $dst\t# int" %}
11282 ins_encode %{
11283 __ negq($dst$$Register);
11284 %}
11285 ins_pipe(ialu_reg);
11286 %}
11287
11288 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
11289 %{
11290 predicate(UseAPX);
11291 match(Set dst (NegL src));
11292 effect(KILL cr);
11293 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11294
11295 format %{ "enegq $dst, $src\t# long ndd" %}
11296 ins_encode %{
11297 __ enegq($dst$$Register, $src$$Register, false);
11298 %}
11299 ins_pipe(ialu_reg);
11300 %}
11301
11302 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
11303 %{
11304 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
11305 effect(KILL cr);
11306 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11307
11308 format %{ "negq $dst\t# long" %}
11309 ins_encode %{
11310 __ negq($dst$$Address);
11311 %}
11312 ins_pipe(ialu_reg);
11313 %}
11314
11315 //----------Multiplication/Division Instructions-------------------------------
11316 // Integer Multiplication Instructions
11317 // Multiply Register
11318
11319 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11320 %{
11321 predicate(!UseAPX);
11322 match(Set dst (MulI dst src));
11323 effect(KILL cr);
11324
11325 ins_cost(300);
11326 format %{ "imull $dst, $src\t# int" %}
11327 ins_encode %{
11328 __ imull($dst$$Register, $src$$Register);
11329 %}
11330 ins_pipe(ialu_reg_reg_alu0);
11331 %}
11332
11333 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11334 %{
11335 predicate(UseAPX);
11336 match(Set dst (MulI src1 src2));
11337 effect(KILL cr);
11338 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11339
11340 ins_cost(300);
11341 format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
11342 ins_encode %{
11343 __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
11344 %}
11345 ins_pipe(ialu_reg_reg_alu0);
11346 %}
11347
11348 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
11349 %{
11350 match(Set dst (MulI src imm));
11351 effect(KILL cr);
11352
11353 ins_cost(300);
11354 format %{ "imull $dst, $src, $imm\t# int" %}
11355 ins_encode %{
11356 __ imull($dst$$Register, $src$$Register, $imm$$constant);
11357 %}
11358 ins_pipe(ialu_reg_reg_alu0);
11359 %}
11360
11361 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
11362 %{
11363 match(Set dst (MulI dst (LoadI src)));
11364 effect(KILL cr);
11365
11366 ins_cost(350);
11367 format %{ "imull $dst, $src\t# int" %}
11368 ins_encode %{
11369 __ imull($dst$$Register, $src$$Address);
11370 %}
11371 ins_pipe(ialu_reg_mem_alu0);
11372 %}
11373
11374 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
11375 %{
11376 match(Set dst (MulI (LoadI src) imm));
11377 effect(KILL cr);
11378
11379 ins_cost(300);
11380 format %{ "imull $dst, $src, $imm\t# int" %}
11381 ins_encode %{
11382 __ imull($dst$$Register, $src$$Address, $imm$$constant);
11383 %}
11384 ins_pipe(ialu_reg_mem_alu0);
11385 %}
11386
11387 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
11388 %{
11389 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
11390 effect(KILL cr, KILL src2);
11391
11392 expand %{ mulI_rReg(dst, src1, cr);
11393 mulI_rReg(src2, src3, cr);
11394 addI_rReg(dst, src2, cr); %}
11395 %}
11396
11397 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11398 %{
11399 predicate(!UseAPX);
11400 match(Set dst (MulL dst src));
11401 effect(KILL cr);
11402
11403 ins_cost(300);
11404 format %{ "imulq $dst, $src\t# long" %}
11405 ins_encode %{
11406 __ imulq($dst$$Register, $src$$Register);
11407 %}
11408 ins_pipe(ialu_reg_reg_alu0);
11409 %}
11410
11411 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11412 %{
11413 predicate(UseAPX);
11414 match(Set dst (MulL src1 src2));
11415 effect(KILL cr);
11416 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11417
11418 ins_cost(300);
11419 format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
11420 ins_encode %{
11421 __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
11422 %}
11423 ins_pipe(ialu_reg_reg_alu0);
11424 %}
11425
11426 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
11427 %{
11428 match(Set dst (MulL src imm));
11429 effect(KILL cr);
11430
11431 ins_cost(300);
11432 format %{ "imulq $dst, $src, $imm\t# long" %}
11433 ins_encode %{
11434 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
11435 %}
11436 ins_pipe(ialu_reg_reg_alu0);
11437 %}
11438
11439 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
11440 %{
11441 match(Set dst (MulL dst (LoadL src)));
11442 effect(KILL cr);
11443
11444 ins_cost(350);
11445 format %{ "imulq $dst, $src\t# long" %}
11446 ins_encode %{
11447 __ imulq($dst$$Register, $src$$Address);
11448 %}
11449 ins_pipe(ialu_reg_mem_alu0);
11450 %}
11451
11452
11453 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
11454 %{
11455 match(Set dst (MulL (LoadL src) imm));
11456 effect(KILL cr);
11457
11458 ins_cost(300);
11459 format %{ "imulq $dst, $src, $imm\t# long" %}
11460 ins_encode %{
11461 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
11462 %}
11463 ins_pipe(ialu_reg_mem_alu0);
11464 %}
11465
11466 instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11467 %{
11468 match(MulHiLoL src rax);
11469 match(MulHiLoL rax src);
11470 effect(KILL cr);
11471
11472 ins_cost(300);
11473 format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
11474 ins_encode %{
11475 __ imulq($src$$Register);
11476 %}
11477 ins_pipe(ialu_reg_reg_alu0);
11478 %}
11479
11480 instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11481 %{
11482 match(UMulHiLoL src rax);
11483 match(UMulHiLoL rax src);
11484 effect(KILL cr);
11485
11486 ins_cost(300);
11487 format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
11488 ins_encode %{
11489 __ mulq($src$$Register);
11490 %}
11491 ins_pipe(ialu_reg_reg_alu0);
11492 %}
11493
11494 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11495 %{
11496 match(Set dst (MulHiL src rax));
11497 effect(USE_KILL rax, KILL cr);
11498
11499 ins_cost(300);
11500 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
11501 ins_encode %{
11502 __ imulq($src$$Register);
11503 %}
11504 ins_pipe(ialu_reg_reg_alu0);
11505 %}
11506
11507 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11508 %{
11509 match(Set dst (UMulHiL src rax));
11510 effect(USE_KILL rax, KILL cr);
11511
11512 ins_cost(300);
11513 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
11514 ins_encode %{
11515 __ mulq($src$$Register);
11516 %}
11517 ins_pipe(ialu_reg_reg_alu0);
11518 %}
11519
11520 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11521 rFlagsReg cr)
11522 %{
11523 match(Set rax (DivI rax div));
11524 effect(KILL rdx, KILL cr);
11525
11526 ins_cost(30*100+10*100); // XXX
11527 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11528 "jne,s normal\n\t"
11529 "xorl rdx, rdx\n\t"
11530 "cmpl $div, -1\n\t"
11531 "je,s done\n"
11532 "normal: cdql\n\t"
11533 "idivl $div\n"
11534 "done:" %}
11535 ins_encode(cdql_enc(div));
11536 ins_pipe(ialu_reg_reg_alu0);
11537 %}
11538
11539 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11540 rFlagsReg cr)
11541 %{
11542 match(Set rax (DivL rax div));
11543 effect(KILL rdx, KILL cr);
11544
11545 ins_cost(30*100+10*100); // XXX
11546 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11547 "cmpq rax, rdx\n\t"
11548 "jne,s normal\n\t"
11549 "xorl rdx, rdx\n\t"
11550 "cmpq $div, -1\n\t"
11551 "je,s done\n"
11552 "normal: cdqq\n\t"
11553 "idivq $div\n"
11554 "done:" %}
11555 ins_encode(cdqq_enc(div));
11556 ins_pipe(ialu_reg_reg_alu0);
11557 %}
11558
11559 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
11560 %{
11561 match(Set rax (UDivI rax div));
11562 effect(KILL rdx, KILL cr);
11563
11564 ins_cost(300);
11565 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
11566 ins_encode %{
11567 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
11568 %}
11569 ins_pipe(ialu_reg_reg_alu0);
11570 %}
11571
11572 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
11573 %{
11574 match(Set rax (UDivL rax div));
11575 effect(KILL rdx, KILL cr);
11576
11577 ins_cost(300);
11578 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
11579 ins_encode %{
11580 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
11581 %}
11582 ins_pipe(ialu_reg_reg_alu0);
11583 %}
11584
11585 // Integer DIVMOD with Register, both quotient and mod results
11586 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11587 rFlagsReg cr)
11588 %{
11589 match(DivModI rax div);
11590 effect(KILL cr);
11591
11592 ins_cost(30*100+10*100); // XXX
11593 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11594 "jne,s normal\n\t"
11595 "xorl rdx, rdx\n\t"
11596 "cmpl $div, -1\n\t"
11597 "je,s done\n"
11598 "normal: cdql\n\t"
11599 "idivl $div\n"
11600 "done:" %}
11601 ins_encode(cdql_enc(div));
11602 ins_pipe(pipe_slow);
11603 %}
11604
11605 // Long DIVMOD with Register, both quotient and mod results
11606 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11607 rFlagsReg cr)
11608 %{
11609 match(DivModL rax div);
11610 effect(KILL cr);
11611
11612 ins_cost(30*100+10*100); // XXX
11613 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11614 "cmpq rax, rdx\n\t"
11615 "jne,s normal\n\t"
11616 "xorl rdx, rdx\n\t"
11617 "cmpq $div, -1\n\t"
11618 "je,s done\n"
11619 "normal: cdqq\n\t"
11620 "idivq $div\n"
11621 "done:" %}
11622 ins_encode(cdqq_enc(div));
11623 ins_pipe(pipe_slow);
11624 %}
11625
11626 // Unsigned integer DIVMOD with Register, both quotient and mod results
11627 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
11628 no_rax_rdx_RegI div, rFlagsReg cr)
11629 %{
11630 match(UDivModI rax div);
11631 effect(TEMP tmp, KILL cr);
11632
11633 ins_cost(300);
11634 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
11635 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
11636 %}
11637 ins_encode %{
11638 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11639 %}
11640 ins_pipe(pipe_slow);
11641 %}
11642
11643 // Unsigned long DIVMOD with Register, both quotient and mod results
11644 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
11645 no_rax_rdx_RegL div, rFlagsReg cr)
11646 %{
11647 match(UDivModL rax div);
11648 effect(TEMP tmp, KILL cr);
11649
11650 ins_cost(300);
11651 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
11652 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
11653 %}
11654 ins_encode %{
11655 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11656 %}
11657 ins_pipe(pipe_slow);
11658 %}
11659
11660 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
11661 rFlagsReg cr)
11662 %{
11663 match(Set rdx (ModI rax div));
11664 effect(KILL rax, KILL cr);
11665
11666 ins_cost(300); // XXX
11667 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
11668 "jne,s normal\n\t"
11669 "xorl rdx, rdx\n\t"
11670 "cmpl $div, -1\n\t"
11671 "je,s done\n"
11672 "normal: cdql\n\t"
11673 "idivl $div\n"
11674 "done:" %}
11675 ins_encode(cdql_enc(div));
11676 ins_pipe(ialu_reg_reg_alu0);
11677 %}
11678
11679 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
11680 rFlagsReg cr)
11681 %{
11682 match(Set rdx (ModL rax div));
11683 effect(KILL rax, KILL cr);
11684
11685 ins_cost(300); // XXX
11686 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
11687 "cmpq rax, rdx\n\t"
11688 "jne,s normal\n\t"
11689 "xorl rdx, rdx\n\t"
11690 "cmpq $div, -1\n\t"
11691 "je,s done\n"
11692 "normal: cdqq\n\t"
11693 "idivq $div\n"
11694 "done:" %}
11695 ins_encode(cdqq_enc(div));
11696 ins_pipe(ialu_reg_reg_alu0);
11697 %}
11698
11699 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
11700 %{
11701 match(Set rdx (UModI rax div));
11702 effect(KILL rax, KILL cr);
11703
11704 ins_cost(300);
11705 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
11706 ins_encode %{
11707 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
11708 %}
11709 ins_pipe(ialu_reg_reg_alu0);
11710 %}
11711
11712 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
11713 %{
11714 match(Set rdx (UModL rax div));
11715 effect(KILL rax, KILL cr);
11716
11717 ins_cost(300);
11718 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
11719 ins_encode %{
11720 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
11721 %}
11722 ins_pipe(ialu_reg_reg_alu0);
11723 %}
11724
11725 // Integer Shift Instructions
11726 // Shift Left by one, two, three
11727 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
11728 %{
11729 predicate(!UseAPX);
11730 match(Set dst (LShiftI dst shift));
11731 effect(KILL cr);
11732
11733 format %{ "sall $dst, $shift" %}
11734 ins_encode %{
11735 __ sall($dst$$Register, $shift$$constant);
11736 %}
11737 ins_pipe(ialu_reg);
11738 %}
11739
11740 // Shift Left by one, two, three
11741 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
11742 %{
11743 predicate(UseAPX);
11744 match(Set dst (LShiftI src shift));
11745 effect(KILL cr);
11746 flag(PD::Flag_ndd_demotable_opr1);
11747
11748 format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
11749 ins_encode %{
11750 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11751 %}
11752 ins_pipe(ialu_reg);
11753 %}
11754
11755 // Shift Left by 8-bit immediate
11756 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11757 %{
11758 predicate(!UseAPX);
11759 match(Set dst (LShiftI dst shift));
11760 effect(KILL cr);
11761
11762 format %{ "sall $dst, $shift" %}
11763 ins_encode %{
11764 __ sall($dst$$Register, $shift$$constant);
11765 %}
11766 ins_pipe(ialu_reg);
11767 %}
11768
11769 // Shift Left by 8-bit immediate
11770 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11771 %{
11772 predicate(UseAPX);
11773 match(Set dst (LShiftI src shift));
11774 effect(KILL cr);
11775 flag(PD::Flag_ndd_demotable_opr1);
11776
11777 format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
11778 ins_encode %{
11779 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11780 %}
11781 ins_pipe(ialu_reg);
11782 %}
11783
11784 // Shift Left by 8-bit immediate
11785 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11786 %{
11787 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11788 effect(KILL cr);
11789
11790 format %{ "sall $dst, $shift" %}
11791 ins_encode %{
11792 __ sall($dst$$Address, $shift$$constant);
11793 %}
11794 ins_pipe(ialu_mem_imm);
11795 %}
11796
11797 // Shift Left by variable
11798 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11799 %{
11800 predicate(!VM_Version::supports_bmi2());
11801 match(Set dst (LShiftI dst shift));
11802 effect(KILL cr);
11803
11804 format %{ "sall $dst, $shift" %}
11805 ins_encode %{
11806 __ sall($dst$$Register);
11807 %}
11808 ins_pipe(ialu_reg_reg);
11809 %}
11810
11811 // Shift Left by variable
11812 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11813 %{
11814 predicate(!VM_Version::supports_bmi2());
11815 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11816 effect(KILL cr);
11817
11818 format %{ "sall $dst, $shift" %}
11819 ins_encode %{
11820 __ sall($dst$$Address);
11821 %}
11822 ins_pipe(ialu_mem_reg);
11823 %}
11824
11825 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11826 %{
11827 predicate(VM_Version::supports_bmi2());
11828 match(Set dst (LShiftI src shift));
11829
11830 format %{ "shlxl $dst, $src, $shift" %}
11831 ins_encode %{
11832 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
11833 %}
11834 ins_pipe(ialu_reg_reg);
11835 %}
11836
11837 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
11838 %{
11839 predicate(VM_Version::supports_bmi2());
11840 match(Set dst (LShiftI (LoadI src) shift));
11841 ins_cost(175);
11842 format %{ "shlxl $dst, $src, $shift" %}
11843 ins_encode %{
11844 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
11845 %}
11846 ins_pipe(ialu_reg_mem);
11847 %}
11848
11849 // Arithmetic Shift Right by 8-bit immediate
11850 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11851 %{
11852 predicate(!UseAPX);
11853 match(Set dst (RShiftI dst shift));
11854 effect(KILL cr);
11855
11856 format %{ "sarl $dst, $shift" %}
11857 ins_encode %{
11858 __ sarl($dst$$Register, $shift$$constant);
11859 %}
11860 ins_pipe(ialu_mem_imm);
11861 %}
11862
11863 // Arithmetic Shift Right by 8-bit immediate
11864 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11865 %{
11866 predicate(UseAPX);
11867 match(Set dst (RShiftI src shift));
11868 effect(KILL cr);
11869 flag(PD::Flag_ndd_demotable_opr1);
11870
11871 format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
11872 ins_encode %{
11873 __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
11874 %}
11875 ins_pipe(ialu_mem_imm);
11876 %}
11877
11878 // Arithmetic Shift Right by 8-bit immediate
11879 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11880 %{
11881 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11882 effect(KILL cr);
11883
11884 format %{ "sarl $dst, $shift" %}
11885 ins_encode %{
11886 __ sarl($dst$$Address, $shift$$constant);
11887 %}
11888 ins_pipe(ialu_mem_imm);
11889 %}
11890
11891 // Arithmetic Shift Right by variable
11892 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11893 %{
11894 predicate(!VM_Version::supports_bmi2());
11895 match(Set dst (RShiftI dst shift));
11896 effect(KILL cr);
11897
11898 format %{ "sarl $dst, $shift" %}
11899 ins_encode %{
11900 __ sarl($dst$$Register);
11901 %}
11902 ins_pipe(ialu_reg_reg);
11903 %}
11904
11905 // Arithmetic Shift Right by variable
11906 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11907 %{
11908 predicate(!VM_Version::supports_bmi2());
11909 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11910 effect(KILL cr);
11911
11912 format %{ "sarl $dst, $shift" %}
11913 ins_encode %{
11914 __ sarl($dst$$Address);
11915 %}
11916 ins_pipe(ialu_mem_reg);
11917 %}
11918
11919 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11920 %{
11921 predicate(VM_Version::supports_bmi2());
11922 match(Set dst (RShiftI src shift));
11923
11924 format %{ "sarxl $dst, $src, $shift" %}
11925 ins_encode %{
11926 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
11927 %}
11928 ins_pipe(ialu_reg_reg);
11929 %}
11930
11931 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
11932 %{
11933 predicate(VM_Version::supports_bmi2());
11934 match(Set dst (RShiftI (LoadI src) shift));
11935 ins_cost(175);
11936 format %{ "sarxl $dst, $src, $shift" %}
11937 ins_encode %{
11938 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
11939 %}
11940 ins_pipe(ialu_reg_mem);
11941 %}
11942
11943 // Logical Shift Right by 8-bit immediate
11944 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11945 %{
11946 predicate(!UseAPX);
11947 match(Set dst (URShiftI dst shift));
11948 effect(KILL cr);
11949
11950 format %{ "shrl $dst, $shift" %}
11951 ins_encode %{
11952 __ shrl($dst$$Register, $shift$$constant);
11953 %}
11954 ins_pipe(ialu_reg);
11955 %}
11956
11957 // Logical Shift Right by 8-bit immediate
11958 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11959 %{
11960 predicate(UseAPX);
11961 match(Set dst (URShiftI src shift));
11962 effect(KILL cr);
11963 flag(PD::Flag_ndd_demotable_opr1);
11964
11965 format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
11966 ins_encode %{
11967 __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
11968 %}
11969 ins_pipe(ialu_reg);
11970 %}
11971
11972 // Logical Shift Right by 8-bit immediate
11973 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11974 %{
11975 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11976 effect(KILL cr);
11977
11978 format %{ "shrl $dst, $shift" %}
11979 ins_encode %{
11980 __ shrl($dst$$Address, $shift$$constant);
11981 %}
11982 ins_pipe(ialu_mem_imm);
11983 %}
11984
11985 // Logical Shift Right by variable
11986 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11987 %{
11988 predicate(!VM_Version::supports_bmi2());
11989 match(Set dst (URShiftI dst shift));
11990 effect(KILL cr);
11991
11992 format %{ "shrl $dst, $shift" %}
11993 ins_encode %{
11994 __ shrl($dst$$Register);
11995 %}
11996 ins_pipe(ialu_reg_reg);
11997 %}
11998
11999 // Logical Shift Right by variable
12000 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12001 %{
12002 predicate(!VM_Version::supports_bmi2());
12003 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
12004 effect(KILL cr);
12005
12006 format %{ "shrl $dst, $shift" %}
12007 ins_encode %{
12008 __ shrl($dst$$Address);
12009 %}
12010 ins_pipe(ialu_mem_reg);
12011 %}
12012
12013 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
12014 %{
12015 predicate(VM_Version::supports_bmi2());
12016 match(Set dst (URShiftI src shift));
12017
12018 format %{ "shrxl $dst, $src, $shift" %}
12019 ins_encode %{
12020 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
12021 %}
12022 ins_pipe(ialu_reg_reg);
12023 %}
12024
12025 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
12026 %{
12027 predicate(VM_Version::supports_bmi2());
12028 match(Set dst (URShiftI (LoadI src) shift));
12029 ins_cost(175);
12030 format %{ "shrxl $dst, $src, $shift" %}
12031 ins_encode %{
12032 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
12033 %}
12034 ins_pipe(ialu_reg_mem);
12035 %}
12036
12037 // Long Shift Instructions
12038 // Shift Left by one, two, three
12039 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
12040 %{
12041 predicate(!UseAPX);
12042 match(Set dst (LShiftL dst shift));
12043 effect(KILL cr);
12044
12045 format %{ "salq $dst, $shift" %}
12046 ins_encode %{
12047 __ salq($dst$$Register, $shift$$constant);
12048 %}
12049 ins_pipe(ialu_reg);
12050 %}
12051
12052 // Shift Left by one, two, three
12053 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
12054 %{
12055 predicate(UseAPX);
12056 match(Set dst (LShiftL src shift));
12057 effect(KILL cr);
12058 flag(PD::Flag_ndd_demotable_opr1);
12059
12060 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12061 ins_encode %{
12062 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12063 %}
12064 ins_pipe(ialu_reg);
12065 %}
12066
12067 // Shift Left by 8-bit immediate
12068 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12069 %{
12070 predicate(!UseAPX);
12071 match(Set dst (LShiftL dst shift));
12072 effect(KILL cr);
12073
12074 format %{ "salq $dst, $shift" %}
12075 ins_encode %{
12076 __ salq($dst$$Register, $shift$$constant);
12077 %}
12078 ins_pipe(ialu_reg);
12079 %}
12080
12081 // Shift Left by 8-bit immediate
12082 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12083 %{
12084 predicate(UseAPX);
12085 match(Set dst (LShiftL src shift));
12086 effect(KILL cr);
12087 flag(PD::Flag_ndd_demotable_opr1);
12088
12089 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12090 ins_encode %{
12091 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12092 %}
12093 ins_pipe(ialu_reg);
12094 %}
12095
12096 // Shift Left by 8-bit immediate
12097 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12098 %{
12099 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12100 effect(KILL cr);
12101
12102 format %{ "salq $dst, $shift" %}
12103 ins_encode %{
12104 __ salq($dst$$Address, $shift$$constant);
12105 %}
12106 ins_pipe(ialu_mem_imm);
12107 %}
12108
12109 // Shift Left by variable
12110 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12111 %{
12112 predicate(!VM_Version::supports_bmi2());
12113 match(Set dst (LShiftL dst shift));
12114 effect(KILL cr);
12115
12116 format %{ "salq $dst, $shift" %}
12117 ins_encode %{
12118 __ salq($dst$$Register);
12119 %}
12120 ins_pipe(ialu_reg_reg);
12121 %}
12122
12123 // Shift Left by variable
12124 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12125 %{
12126 predicate(!VM_Version::supports_bmi2());
12127 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12128 effect(KILL cr);
12129
12130 format %{ "salq $dst, $shift" %}
12131 ins_encode %{
12132 __ salq($dst$$Address);
12133 %}
12134 ins_pipe(ialu_mem_reg);
12135 %}
12136
12137 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12138 %{
12139 predicate(VM_Version::supports_bmi2());
12140 match(Set dst (LShiftL src shift));
12141
12142 format %{ "shlxq $dst, $src, $shift" %}
12143 ins_encode %{
12144 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
12145 %}
12146 ins_pipe(ialu_reg_reg);
12147 %}
12148
12149 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
12150 %{
12151 predicate(VM_Version::supports_bmi2());
12152 match(Set dst (LShiftL (LoadL src) shift));
12153 ins_cost(175);
12154 format %{ "shlxq $dst, $src, $shift" %}
12155 ins_encode %{
12156 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
12157 %}
12158 ins_pipe(ialu_reg_mem);
12159 %}
12160
12161 // Arithmetic Shift Right by 8-bit immediate
12162 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
12163 %{
12164 predicate(!UseAPX);
12165 match(Set dst (RShiftL dst shift));
12166 effect(KILL cr);
12167
12168 format %{ "sarq $dst, $shift" %}
12169 ins_encode %{
12170 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
12171 %}
12172 ins_pipe(ialu_mem_imm);
12173 %}
12174
12175 // Arithmetic Shift Right by 8-bit immediate
12176 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
12177 %{
12178 predicate(UseAPX);
12179 match(Set dst (RShiftL src shift));
12180 effect(KILL cr);
12181 flag(PD::Flag_ndd_demotable_opr1);
12182
12183 format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
12184 ins_encode %{
12185 __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
12186 %}
12187 ins_pipe(ialu_mem_imm);
12188 %}
12189
12190 // Arithmetic Shift Right by 8-bit immediate
12191 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
12192 %{
12193 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12194 effect(KILL cr);
12195
12196 format %{ "sarq $dst, $shift" %}
12197 ins_encode %{
12198 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
12199 %}
12200 ins_pipe(ialu_mem_imm);
12201 %}
12202
12203 // Arithmetic Shift Right by variable
12204 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12205 %{
12206 predicate(!VM_Version::supports_bmi2());
12207 match(Set dst (RShiftL dst shift));
12208 effect(KILL cr);
12209
12210 format %{ "sarq $dst, $shift" %}
12211 ins_encode %{
12212 __ sarq($dst$$Register);
12213 %}
12214 ins_pipe(ialu_reg_reg);
12215 %}
12216
12217 // Arithmetic Shift Right by variable
12218 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12219 %{
12220 predicate(!VM_Version::supports_bmi2());
12221 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12222 effect(KILL cr);
12223
12224 format %{ "sarq $dst, $shift" %}
12225 ins_encode %{
12226 __ sarq($dst$$Address);
12227 %}
12228 ins_pipe(ialu_mem_reg);
12229 %}
12230
12231 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12232 %{
12233 predicate(VM_Version::supports_bmi2());
12234 match(Set dst (RShiftL src shift));
12235
12236 format %{ "sarxq $dst, $src, $shift" %}
12237 ins_encode %{
12238 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
12239 %}
12240 ins_pipe(ialu_reg_reg);
12241 %}
12242
12243 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
12244 %{
12245 predicate(VM_Version::supports_bmi2());
12246 match(Set dst (RShiftL (LoadL src) shift));
12247 ins_cost(175);
12248 format %{ "sarxq $dst, $src, $shift" %}
12249 ins_encode %{
12250 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
12251 %}
12252 ins_pipe(ialu_reg_mem);
12253 %}
12254
12255 // Logical Shift Right by 8-bit immediate
12256 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12257 %{
12258 predicate(!UseAPX);
12259 match(Set dst (URShiftL dst shift));
12260 effect(KILL cr);
12261
12262 format %{ "shrq $dst, $shift" %}
12263 ins_encode %{
12264 __ shrq($dst$$Register, $shift$$constant);
12265 %}
12266 ins_pipe(ialu_reg);
12267 %}
12268
12269 // Logical Shift Right by 8-bit immediate
12270 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12271 %{
12272 predicate(UseAPX);
12273 match(Set dst (URShiftL src shift));
12274 effect(KILL cr);
12275 flag(PD::Flag_ndd_demotable_opr1);
12276
12277 format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
12278 ins_encode %{
12279 __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
12280 %}
12281 ins_pipe(ialu_reg);
12282 %}
12283
12284 // Logical Shift Right by 8-bit immediate
12285 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12286 %{
12287 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12288 effect(KILL cr);
12289
12290 format %{ "shrq $dst, $shift" %}
12291 ins_encode %{
12292 __ shrq($dst$$Address, $shift$$constant);
12293 %}
12294 ins_pipe(ialu_mem_imm);
12295 %}
12296
12297 // Logical Shift Right by variable
12298 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12299 %{
12300 predicate(!VM_Version::supports_bmi2());
12301 match(Set dst (URShiftL dst shift));
12302 effect(KILL cr);
12303
12304 format %{ "shrq $dst, $shift" %}
12305 ins_encode %{
12306 __ shrq($dst$$Register);
12307 %}
12308 ins_pipe(ialu_reg_reg);
12309 %}
12310
12311 // Logical Shift Right by variable
12312 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12313 %{
12314 predicate(!VM_Version::supports_bmi2());
12315 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12316 effect(KILL cr);
12317
12318 format %{ "shrq $dst, $shift" %}
12319 ins_encode %{
12320 __ shrq($dst$$Address);
12321 %}
12322 ins_pipe(ialu_mem_reg);
12323 %}
12324
12325 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12326 %{
12327 predicate(VM_Version::supports_bmi2());
12328 match(Set dst (URShiftL src shift));
12329
12330 format %{ "shrxq $dst, $src, $shift" %}
12331 ins_encode %{
12332 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
12333 %}
12334 ins_pipe(ialu_reg_reg);
12335 %}
12336
12337 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
12338 %{
12339 predicate(VM_Version::supports_bmi2());
12340 match(Set dst (URShiftL (LoadL src) shift));
12341 ins_cost(175);
12342 format %{ "shrxq $dst, $src, $shift" %}
12343 ins_encode %{
12344 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
12345 %}
12346 ins_pipe(ialu_reg_mem);
12347 %}
12348
12349 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
12350 // This idiom is used by the compiler for the i2b bytecode.
12351 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
12352 %{
12353 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
12354
12355 format %{ "movsbl $dst, $src\t# i2b" %}
12356 ins_encode %{
12357 __ movsbl($dst$$Register, $src$$Register);
12358 %}
12359 ins_pipe(ialu_reg_reg);
12360 %}
12361
12362 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
12363 // This idiom is used by the compiler the i2s bytecode.
12364 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
12365 %{
12366 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
12367
12368 format %{ "movswl $dst, $src\t# i2s" %}
12369 ins_encode %{
12370 __ movswl($dst$$Register, $src$$Register);
12371 %}
12372 ins_pipe(ialu_reg_reg);
12373 %}
12374
12375 // ROL/ROR instructions
12376
12377 // Rotate left by constant.
12378 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12379 %{
12380 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12381 match(Set dst (RotateLeft dst shift));
12382 effect(KILL cr);
12383 format %{ "roll $dst, $shift" %}
12384 ins_encode %{
12385 __ roll($dst$$Register, $shift$$constant);
12386 %}
12387 ins_pipe(ialu_reg);
12388 %}
12389
12390 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
12391 %{
12392 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12393 match(Set dst (RotateLeft src shift));
12394 format %{ "rolxl $dst, $src, $shift" %}
12395 ins_encode %{
12396 int shift = 32 - ($shift$$constant & 31);
12397 __ rorxl($dst$$Register, $src$$Register, shift);
12398 %}
12399 ins_pipe(ialu_reg_reg);
12400 %}
12401
12402 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
12403 %{
12404 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12405 match(Set dst (RotateLeft (LoadI src) shift));
12406 ins_cost(175);
12407 format %{ "rolxl $dst, $src, $shift" %}
12408 ins_encode %{
12409 int shift = 32 - ($shift$$constant & 31);
12410 __ rorxl($dst$$Register, $src$$Address, shift);
12411 %}
12412 ins_pipe(ialu_reg_mem);
12413 %}
12414
12415 // Rotate Left by variable
12416 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12417 %{
12418 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12419 match(Set dst (RotateLeft dst shift));
12420 effect(KILL cr);
12421 format %{ "roll $dst, $shift" %}
12422 ins_encode %{
12423 __ roll($dst$$Register);
12424 %}
12425 ins_pipe(ialu_reg_reg);
12426 %}
12427
12428 // Rotate Left by variable
12429 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12430 %{
12431 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12432 match(Set dst (RotateLeft src shift));
12433 effect(KILL cr);
12434 flag(PD::Flag_ndd_demotable_opr1);
12435
12436 format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
12437 ins_encode %{
12438 __ eroll($dst$$Register, $src$$Register, false);
12439 %}
12440 ins_pipe(ialu_reg_reg);
12441 %}
12442
12443 // Rotate Right by constant.
12444 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12445 %{
12446 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12447 match(Set dst (RotateRight dst shift));
12448 effect(KILL cr);
12449 format %{ "rorl $dst, $shift" %}
12450 ins_encode %{
12451 __ rorl($dst$$Register, $shift$$constant);
12452 %}
12453 ins_pipe(ialu_reg);
12454 %}
12455
12456 // Rotate Right by constant.
12457 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
12458 %{
12459 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12460 match(Set dst (RotateRight src shift));
12461 format %{ "rorxl $dst, $src, $shift" %}
12462 ins_encode %{
12463 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
12464 %}
12465 ins_pipe(ialu_reg_reg);
12466 %}
12467
12468 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
12469 %{
12470 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12471 match(Set dst (RotateRight (LoadI src) shift));
12472 ins_cost(175);
12473 format %{ "rorxl $dst, $src, $shift" %}
12474 ins_encode %{
12475 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
12476 %}
12477 ins_pipe(ialu_reg_mem);
12478 %}
12479
12480 // Rotate Right by variable
12481 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12482 %{
12483 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12484 match(Set dst (RotateRight dst shift));
12485 effect(KILL cr);
12486 format %{ "rorl $dst, $shift" %}
12487 ins_encode %{
12488 __ rorl($dst$$Register);
12489 %}
12490 ins_pipe(ialu_reg_reg);
12491 %}
12492
12493 // Rotate Right by variable
12494 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12495 %{
12496 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12497 match(Set dst (RotateRight src shift));
12498 effect(KILL cr);
12499 flag(PD::Flag_ndd_demotable_opr1);
12500
12501 format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
12502 ins_encode %{
12503 __ erorl($dst$$Register, $src$$Register, false);
12504 %}
12505 ins_pipe(ialu_reg_reg);
12506 %}
12507
12508 // Rotate Left by constant.
12509 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12510 %{
12511 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12512 match(Set dst (RotateLeft dst shift));
12513 effect(KILL cr);
12514 format %{ "rolq $dst, $shift" %}
12515 ins_encode %{
12516 __ rolq($dst$$Register, $shift$$constant);
12517 %}
12518 ins_pipe(ialu_reg);
12519 %}
12520
12521 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
12522 %{
12523 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12524 match(Set dst (RotateLeft src shift));
12525 format %{ "rolxq $dst, $src, $shift" %}
12526 ins_encode %{
12527 int shift = 64 - ($shift$$constant & 63);
12528 __ rorxq($dst$$Register, $src$$Register, shift);
12529 %}
12530 ins_pipe(ialu_reg_reg);
12531 %}
12532
12533 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
12534 %{
12535 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12536 match(Set dst (RotateLeft (LoadL src) shift));
12537 ins_cost(175);
12538 format %{ "rolxq $dst, $src, $shift" %}
12539 ins_encode %{
12540 int shift = 64 - ($shift$$constant & 63);
12541 __ rorxq($dst$$Register, $src$$Address, shift);
12542 %}
12543 ins_pipe(ialu_reg_mem);
12544 %}
12545
12546 // Rotate Left by variable
12547 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12548 %{
12549 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12550 match(Set dst (RotateLeft dst shift));
12551 effect(KILL cr);
12552
12553 format %{ "rolq $dst, $shift" %}
12554 ins_encode %{
12555 __ rolq($dst$$Register);
12556 %}
12557 ins_pipe(ialu_reg_reg);
12558 %}
12559
12560 // Rotate Left by variable
12561 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12562 %{
12563 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12564 match(Set dst (RotateLeft src shift));
12565 effect(KILL cr);
12566 flag(PD::Flag_ndd_demotable_opr1);
12567
12568 format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
12569 ins_encode %{
12570 __ erolq($dst$$Register, $src$$Register, false);
12571 %}
12572 ins_pipe(ialu_reg_reg);
12573 %}
12574
12575 // Rotate Right by constant.
12576 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12577 %{
12578 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12579 match(Set dst (RotateRight dst shift));
12580 effect(KILL cr);
12581 format %{ "rorq $dst, $shift" %}
12582 ins_encode %{
12583 __ rorq($dst$$Register, $shift$$constant);
12584 %}
12585 ins_pipe(ialu_reg);
12586 %}
12587
12588 // Rotate Right by constant
12589 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
12590 %{
12591 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12592 match(Set dst (RotateRight src shift));
12593 format %{ "rorxq $dst, $src, $shift" %}
12594 ins_encode %{
12595 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
12596 %}
12597 ins_pipe(ialu_reg_reg);
12598 %}
12599
12600 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
12601 %{
12602 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12603 match(Set dst (RotateRight (LoadL src) shift));
12604 ins_cost(175);
12605 format %{ "rorxq $dst, $src, $shift" %}
12606 ins_encode %{
12607 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
12608 %}
12609 ins_pipe(ialu_reg_mem);
12610 %}
12611
12612 // Rotate Right by variable
12613 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12614 %{
12615 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12616 match(Set dst (RotateRight dst shift));
12617 effect(KILL cr);
12618 format %{ "rorq $dst, $shift" %}
12619 ins_encode %{
12620 __ rorq($dst$$Register);
12621 %}
12622 ins_pipe(ialu_reg_reg);
12623 %}
12624
12625 // Rotate Right by variable
12626 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12627 %{
12628 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12629 match(Set dst (RotateRight src shift));
12630 effect(KILL cr);
12631 flag(PD::Flag_ndd_demotable_opr1);
12632
12633 format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
12634 ins_encode %{
12635 __ erorq($dst$$Register, $src$$Register, false);
12636 %}
12637 ins_pipe(ialu_reg_reg);
12638 %}
12639
12640 //----------------------------- CompressBits/ExpandBits ------------------------
12641
12642 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12643 predicate(n->bottom_type()->isa_long());
12644 match(Set dst (CompressBits src mask));
12645 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12646 ins_encode %{
12647 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
12648 %}
12649 ins_pipe( pipe_slow );
12650 %}
12651
12652 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12653 predicate(n->bottom_type()->isa_long());
12654 match(Set dst (ExpandBits src mask));
12655 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12656 ins_encode %{
12657 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
12658 %}
12659 ins_pipe( pipe_slow );
12660 %}
12661
12662 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12663 predicate(n->bottom_type()->isa_long());
12664 match(Set dst (CompressBits src (LoadL mask)));
12665 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12666 ins_encode %{
12667 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
12668 %}
12669 ins_pipe( pipe_slow );
12670 %}
12671
12672 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12673 predicate(n->bottom_type()->isa_long());
12674 match(Set dst (ExpandBits src (LoadL mask)));
12675 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12676 ins_encode %{
12677 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
12678 %}
12679 ins_pipe( pipe_slow );
12680 %}
12681
12682
12683 // Logical Instructions
12684
12685 // Integer Logical Instructions
12686
12687 // And Instructions
12688 // And Register with Register
12689 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12690 %{
12691 predicate(!UseAPX);
12692 match(Set dst (AndI dst src));
12693 effect(KILL cr);
12694 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12695
12696 format %{ "andl $dst, $src\t# int" %}
12697 ins_encode %{
12698 __ andl($dst$$Register, $src$$Register);
12699 %}
12700 ins_pipe(ialu_reg_reg);
12701 %}
12702
12703 // And Register with Register using New Data Destination (NDD)
12704 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12705 %{
12706 predicate(UseAPX);
12707 match(Set dst (AndI src1 src2));
12708 effect(KILL cr);
12709 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12710
12711 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12712 ins_encode %{
12713 __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
12714
12715 %}
12716 ins_pipe(ialu_reg_reg);
12717 %}
12718
12719 // And Register with Immediate 255
12720 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
12721 %{
12722 match(Set dst (AndI src mask));
12723
12724 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
12725 ins_encode %{
12726 __ movzbl($dst$$Register, $src$$Register);
12727 %}
12728 ins_pipe(ialu_reg);
12729 %}
12730
12731 // And Register with Immediate 255 and promote to long
12732 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
12733 %{
12734 match(Set dst (ConvI2L (AndI src mask)));
12735
12736 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
12737 ins_encode %{
12738 __ movzbl($dst$$Register, $src$$Register);
12739 %}
12740 ins_pipe(ialu_reg);
12741 %}
12742
12743 // And Register with Immediate 65535
12744 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
12745 %{
12746 match(Set dst (AndI src mask));
12747
12748 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
12749 ins_encode %{
12750 __ movzwl($dst$$Register, $src$$Register);
12751 %}
12752 ins_pipe(ialu_reg);
12753 %}
12754
12755 // And Register with Immediate 65535 and promote to long
12756 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
12757 %{
12758 match(Set dst (ConvI2L (AndI src mask)));
12759
12760 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
12761 ins_encode %{
12762 __ movzwl($dst$$Register, $src$$Register);
12763 %}
12764 ins_pipe(ialu_reg);
12765 %}
12766
12767 // Can skip int2long conversions after AND with small bitmask
12768 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
12769 %{
12770 predicate(VM_Version::supports_bmi2());
12771 ins_cost(125);
12772 effect(TEMP tmp, KILL cr);
12773 match(Set dst (ConvI2L (AndI src mask)));
12774 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
12775 ins_encode %{
12776 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
12777 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
12778 %}
12779 ins_pipe(ialu_reg_reg);
12780 %}
12781
12782 // And Register with Immediate
12783 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12784 %{
12785 predicate(!UseAPX);
12786 match(Set dst (AndI dst src));
12787 effect(KILL cr);
12788 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12789
12790 format %{ "andl $dst, $src\t# int" %}
12791 ins_encode %{
12792 __ andl($dst$$Register, $src$$constant);
12793 %}
12794 ins_pipe(ialu_reg);
12795 %}
12796
12797 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12798 %{
12799 predicate(UseAPX);
12800 match(Set dst (AndI src1 src2));
12801 effect(KILL cr);
12802 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12803
12804 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12805 ins_encode %{
12806 __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
12807 %}
12808 ins_pipe(ialu_reg);
12809 %}
12810
12811 // And Register with Memory
12812 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12813 %{
12814 match(Set dst (AndI dst (LoadI src)));
12815 effect(KILL cr);
12816 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12817
12818 ins_cost(150);
12819 format %{ "andl $dst, $src\t# int" %}
12820 ins_encode %{
12821 __ andl($dst$$Register, $src$$Address);
12822 %}
12823 ins_pipe(ialu_reg_mem);
12824 %}
12825
12826 // And Memory with Register
12827 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12828 %{
12829 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
12830 effect(KILL cr);
12831 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12832
12833 ins_cost(150);
12834 format %{ "andb $dst, $src\t# byte" %}
12835 ins_encode %{
12836 __ andb($dst$$Address, $src$$Register);
12837 %}
12838 ins_pipe(ialu_mem_reg);
12839 %}
12840
12841 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12842 %{
12843 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12844 effect(KILL cr);
12845 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12846
12847 ins_cost(150);
12848 format %{ "andl $dst, $src\t# int" %}
12849 ins_encode %{
12850 __ andl($dst$$Address, $src$$Register);
12851 %}
12852 ins_pipe(ialu_mem_reg);
12853 %}
12854
12855 // And Memory with Immediate
12856 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
12857 %{
12858 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12859 effect(KILL cr);
12860 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12861
12862 ins_cost(125);
12863 format %{ "andl $dst, $src\t# int" %}
12864 ins_encode %{
12865 __ andl($dst$$Address, $src$$constant);
12866 %}
12867 ins_pipe(ialu_mem_imm);
12868 %}
12869
12870 // BMI1 instructions
12871 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
12872 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
12873 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12874 effect(KILL cr);
12875 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12876
12877 ins_cost(125);
12878 format %{ "andnl $dst, $src1, $src2" %}
12879
12880 ins_encode %{
12881 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
12882 %}
12883 ins_pipe(ialu_reg_mem);
12884 %}
12885
12886 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
12887 match(Set dst (AndI (XorI src1 minus_1) src2));
12888 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12889 effect(KILL cr);
12890 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12891
12892 format %{ "andnl $dst, $src1, $src2" %}
12893
12894 ins_encode %{
12895 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
12896 %}
12897 ins_pipe(ialu_reg);
12898 %}
12899
12900 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
12901 match(Set dst (AndI (SubI imm_zero src) src));
12902 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12903 effect(KILL cr);
12904 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12905
12906 format %{ "blsil $dst, $src" %}
12907
12908 ins_encode %{
12909 __ blsil($dst$$Register, $src$$Register);
12910 %}
12911 ins_pipe(ialu_reg);
12912 %}
12913
12914 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
12915 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
12916 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12917 effect(KILL cr);
12918 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12919
12920 ins_cost(125);
12921 format %{ "blsil $dst, $src" %}
12922
12923 ins_encode %{
12924 __ blsil($dst$$Register, $src$$Address);
12925 %}
12926 ins_pipe(ialu_reg_mem);
12927 %}
12928
12929 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12930 %{
12931 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
12932 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12933 effect(KILL cr);
12934 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12935
12936 ins_cost(125);
12937 format %{ "blsmskl $dst, $src" %}
12938
12939 ins_encode %{
12940 __ blsmskl($dst$$Register, $src$$Address);
12941 %}
12942 ins_pipe(ialu_reg_mem);
12943 %}
12944
12945 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12946 %{
12947 match(Set dst (XorI (AddI src minus_1) src));
12948 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12949 effect(KILL cr);
12950 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12951
12952 format %{ "blsmskl $dst, $src" %}
12953
12954 ins_encode %{
12955 __ blsmskl($dst$$Register, $src$$Register);
12956 %}
12957
12958 ins_pipe(ialu_reg);
12959 %}
12960
12961 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12962 %{
12963 match(Set dst (AndI (AddI src minus_1) src) );
12964 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12965 effect(KILL cr);
12966 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12967
12968 format %{ "blsrl $dst, $src" %}
12969
12970 ins_encode %{
12971 __ blsrl($dst$$Register, $src$$Register);
12972 %}
12973
12974 ins_pipe(ialu_reg_mem);
12975 %}
12976
12977 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12978 %{
12979 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
12980 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12981 effect(KILL cr);
12982 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12983
12984 ins_cost(125);
12985 format %{ "blsrl $dst, $src" %}
12986
12987 ins_encode %{
12988 __ blsrl($dst$$Register, $src$$Address);
12989 %}
12990
12991 ins_pipe(ialu_reg);
12992 %}
12993
12994 // Or Instructions
12995 // Or Register with Register
12996 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12997 %{
12998 predicate(!UseAPX);
12999 match(Set dst (OrI dst src));
13000 effect(KILL cr);
13001 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13002
13003 format %{ "orl $dst, $src\t# int" %}
13004 ins_encode %{
13005 __ orl($dst$$Register, $src$$Register);
13006 %}
13007 ins_pipe(ialu_reg_reg);
13008 %}
13009
13010 // Or Register with Register using New Data Destination (NDD)
13011 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13012 %{
13013 predicate(UseAPX);
13014 match(Set dst (OrI src1 src2));
13015 effect(KILL cr);
13016 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13017
13018 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
13019 ins_encode %{
13020 __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13021 %}
13022 ins_pipe(ialu_reg_reg);
13023 %}
13024
13025 // Or Register with Immediate
13026 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13027 %{
13028 predicate(!UseAPX);
13029 match(Set dst (OrI dst src));
13030 effect(KILL cr);
13031 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13032
13033 format %{ "orl $dst, $src\t# int" %}
13034 ins_encode %{
13035 __ orl($dst$$Register, $src$$constant);
13036 %}
13037 ins_pipe(ialu_reg);
13038 %}
13039
13040 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13041 %{
13042 predicate(UseAPX);
13043 match(Set dst (OrI src1 src2));
13044 effect(KILL cr);
13045 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13046
13047 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
13048 ins_encode %{
13049 __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13050 %}
13051 ins_pipe(ialu_reg);
13052 %}
13053
13054 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
13055 %{
13056 predicate(UseAPX);
13057 match(Set dst (OrI src1 src2));
13058 effect(KILL cr);
13059 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13060
13061 format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
13062 ins_encode %{
13063 __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
13064 %}
13065 ins_pipe(ialu_reg);
13066 %}
13067
13068 // Or Register with Memory
13069 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13070 %{
13071 match(Set dst (OrI dst (LoadI src)));
13072 effect(KILL cr);
13073 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13074
13075 ins_cost(150);
13076 format %{ "orl $dst, $src\t# int" %}
13077 ins_encode %{
13078 __ orl($dst$$Register, $src$$Address);
13079 %}
13080 ins_pipe(ialu_reg_mem);
13081 %}
13082
13083 // Or Memory with Register
13084 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13085 %{
13086 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
13087 effect(KILL cr);
13088 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13089
13090 ins_cost(150);
13091 format %{ "orb $dst, $src\t# byte" %}
13092 ins_encode %{
13093 __ orb($dst$$Address, $src$$Register);
13094 %}
13095 ins_pipe(ialu_mem_reg);
13096 %}
13097
13098 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13099 %{
13100 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13101 effect(KILL cr);
13102 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13103
13104 ins_cost(150);
13105 format %{ "orl $dst, $src\t# int" %}
13106 ins_encode %{
13107 __ orl($dst$$Address, $src$$Register);
13108 %}
13109 ins_pipe(ialu_mem_reg);
13110 %}
13111
13112 // Or Memory with Immediate
13113 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
13114 %{
13115 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13116 effect(KILL cr);
13117 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13118
13119 ins_cost(125);
13120 format %{ "orl $dst, $src\t# int" %}
13121 ins_encode %{
13122 __ orl($dst$$Address, $src$$constant);
13123 %}
13124 ins_pipe(ialu_mem_imm);
13125 %}
13126
13127 // Xor Instructions
13128 // Xor Register with Register
13129 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
13130 %{
13131 predicate(!UseAPX);
13132 match(Set dst (XorI dst src));
13133 effect(KILL cr);
13134 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13135
13136 format %{ "xorl $dst, $src\t# int" %}
13137 ins_encode %{
13138 __ xorl($dst$$Register, $src$$Register);
13139 %}
13140 ins_pipe(ialu_reg_reg);
13141 %}
13142
13143 // Xor Register with Register using New Data Destination (NDD)
13144 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13145 %{
13146 predicate(UseAPX);
13147 match(Set dst (XorI src1 src2));
13148 effect(KILL cr);
13149 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13150
13151 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13152 ins_encode %{
13153 __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13154 %}
13155 ins_pipe(ialu_reg_reg);
13156 %}
13157
13158 // Xor Register with Immediate -1
13159 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
13160 %{
13161 predicate(!UseAPX);
13162 match(Set dst (XorI dst imm));
13163
13164 format %{ "notl $dst" %}
13165 ins_encode %{
13166 __ notl($dst$$Register);
13167 %}
13168 ins_pipe(ialu_reg);
13169 %}
13170
13171 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
13172 %{
13173 match(Set dst (XorI src imm));
13174 predicate(UseAPX);
13175 flag(PD::Flag_ndd_demotable_opr1);
13176
13177 format %{ "enotl $dst, $src" %}
13178 ins_encode %{
13179 __ enotl($dst$$Register, $src$$Register);
13180 %}
13181 ins_pipe(ialu_reg);
13182 %}
13183
13184 // Xor Register with Immediate
13185 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13186 %{
13187 // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
13188 predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13189 match(Set dst (XorI dst src));
13190 effect(KILL cr);
13191 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13192
13193 format %{ "xorl $dst, $src\t# int" %}
13194 ins_encode %{
13195 __ xorl($dst$$Register, $src$$constant);
13196 %}
13197 ins_pipe(ialu_reg);
13198 %}
13199
13200 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13201 %{
13202 // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
13203 predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13204 match(Set dst (XorI src1 src2));
13205 effect(KILL cr);
13206 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13207
13208 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13209 ins_encode %{
13210 __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13211 %}
13212 ins_pipe(ialu_reg);
13213 %}
13214
13215 // Xor Register with Memory
13216 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13217 %{
13218 match(Set dst (XorI dst (LoadI src)));
13219 effect(KILL cr);
13220 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13221
13222 ins_cost(150);
13223 format %{ "xorl $dst, $src\t# int" %}
13224 ins_encode %{
13225 __ xorl($dst$$Register, $src$$Address);
13226 %}
13227 ins_pipe(ialu_reg_mem);
13228 %}
13229
13230 // Xor Memory with Register
13231 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13232 %{
13233 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
13234 effect(KILL cr);
13235 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13236
13237 ins_cost(150);
13238 format %{ "xorb $dst, $src\t# byte" %}
13239 ins_encode %{
13240 __ xorb($dst$$Address, $src$$Register);
13241 %}
13242 ins_pipe(ialu_mem_reg);
13243 %}
13244
13245 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13246 %{
13247 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13248 effect(KILL cr);
13249 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13250
13251 ins_cost(150);
13252 format %{ "xorl $dst, $src\t# int" %}
13253 ins_encode %{
13254 __ xorl($dst$$Address, $src$$Register);
13255 %}
13256 ins_pipe(ialu_mem_reg);
13257 %}
13258
13259 // Xor Memory with Immediate
13260 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
13261 %{
13262 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13263 effect(KILL cr);
13264 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13265
13266 ins_cost(125);
13267 format %{ "xorl $dst, $src\t# int" %}
13268 ins_encode %{
13269 __ xorl($dst$$Address, $src$$constant);
13270 %}
13271 ins_pipe(ialu_mem_imm);
13272 %}
13273
13274
13275 // Long Logical Instructions
13276
13277 // And Instructions
13278 // And Register with Register
13279 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13280 %{
13281 predicate(!UseAPX);
13282 match(Set dst (AndL dst src));
13283 effect(KILL cr);
13284 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13285
13286 format %{ "andq $dst, $src\t# long" %}
13287 ins_encode %{
13288 __ andq($dst$$Register, $src$$Register);
13289 %}
13290 ins_pipe(ialu_reg_reg);
13291 %}
13292
13293 // And Register with Register using New Data Destination (NDD)
13294 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13295 %{
13296 predicate(UseAPX);
13297 match(Set dst (AndL src1 src2));
13298 effect(KILL cr);
13299 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13300
13301 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13302 ins_encode %{
13303 __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
13304
13305 %}
13306 ins_pipe(ialu_reg_reg);
13307 %}
13308
13309 // And Register with Immediate 255
13310 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
13311 %{
13312 match(Set dst (AndL src mask));
13313
13314 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
13315 ins_encode %{
13316 // movzbl zeroes out the upper 32-bit and does not need REX.W
13317 __ movzbl($dst$$Register, $src$$Register);
13318 %}
13319 ins_pipe(ialu_reg);
13320 %}
13321
13322 // And Register with Immediate 65535
13323 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
13324 %{
13325 match(Set dst (AndL src mask));
13326
13327 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
13328 ins_encode %{
13329 // movzwl zeroes out the upper 32-bit and does not need REX.W
13330 __ movzwl($dst$$Register, $src$$Register);
13331 %}
13332 ins_pipe(ialu_reg);
13333 %}
13334
13335 // And Register with Immediate
13336 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13337 %{
13338 predicate(!UseAPX);
13339 match(Set dst (AndL dst src));
13340 effect(KILL cr);
13341 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13342
13343 format %{ "andq $dst, $src\t# long" %}
13344 ins_encode %{
13345 __ andq($dst$$Register, $src$$constant);
13346 %}
13347 ins_pipe(ialu_reg);
13348 %}
13349
13350 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13351 %{
13352 predicate(UseAPX);
13353 match(Set dst (AndL src1 src2));
13354 effect(KILL cr);
13355 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13356
13357 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13358 ins_encode %{
13359 __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
13360 %}
13361 ins_pipe(ialu_reg);
13362 %}
13363
13364 // And Register with Memory
13365 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13366 %{
13367 match(Set dst (AndL dst (LoadL src)));
13368 effect(KILL cr);
13369 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13370
13371 ins_cost(150);
13372 format %{ "andq $dst, $src\t# long" %}
13373 ins_encode %{
13374 __ andq($dst$$Register, $src$$Address);
13375 %}
13376 ins_pipe(ialu_reg_mem);
13377 %}
13378
13379 // And Memory with Register
13380 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13381 %{
13382 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13383 effect(KILL cr);
13384 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13385
13386 ins_cost(150);
13387 format %{ "andq $dst, $src\t# long" %}
13388 ins_encode %{
13389 __ andq($dst$$Address, $src$$Register);
13390 %}
13391 ins_pipe(ialu_mem_reg);
13392 %}
13393
13394 // And Memory with Immediate
13395 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13396 %{
13397 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13398 effect(KILL cr);
13399 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13400
13401 ins_cost(125);
13402 format %{ "andq $dst, $src\t# long" %}
13403 ins_encode %{
13404 __ andq($dst$$Address, $src$$constant);
13405 %}
13406 ins_pipe(ialu_mem_imm);
13407 %}
13408
13409 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
13410 %{
13411 // con should be a pure 64-bit immediate given that not(con) is a power of 2
13412 // because AND/OR works well enough for 8/32-bit values.
13413 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
13414
13415 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
13416 effect(KILL cr);
13417
13418 ins_cost(125);
13419 format %{ "btrq $dst, log2(not($con))\t# long" %}
13420 ins_encode %{
13421 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
13422 %}
13423 ins_pipe(ialu_mem_imm);
13424 %}
13425
13426 // BMI1 instructions
13427 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
13428 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
13429 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13430 effect(KILL cr);
13431 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13432
13433 ins_cost(125);
13434 format %{ "andnq $dst, $src1, $src2" %}
13435
13436 ins_encode %{
13437 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
13438 %}
13439 ins_pipe(ialu_reg_mem);
13440 %}
13441
13442 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
13443 match(Set dst (AndL (XorL src1 minus_1) src2));
13444 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13445 effect(KILL cr);
13446 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13447
13448 format %{ "andnq $dst, $src1, $src2" %}
13449
13450 ins_encode %{
13451 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
13452 %}
13453 ins_pipe(ialu_reg_mem);
13454 %}
13455
13456 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
13457 match(Set dst (AndL (SubL imm_zero src) src));
13458 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13459 effect(KILL cr);
13460 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13461
13462 format %{ "blsiq $dst, $src" %}
13463
13464 ins_encode %{
13465 __ blsiq($dst$$Register, $src$$Register);
13466 %}
13467 ins_pipe(ialu_reg);
13468 %}
13469
13470 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
13471 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
13472 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13473 effect(KILL cr);
13474 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13475
13476 ins_cost(125);
13477 format %{ "blsiq $dst, $src" %}
13478
13479 ins_encode %{
13480 __ blsiq($dst$$Register, $src$$Address);
13481 %}
13482 ins_pipe(ialu_reg_mem);
13483 %}
13484
13485 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13486 %{
13487 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
13488 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13489 effect(KILL cr);
13490 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13491
13492 ins_cost(125);
13493 format %{ "blsmskq $dst, $src" %}
13494
13495 ins_encode %{
13496 __ blsmskq($dst$$Register, $src$$Address);
13497 %}
13498 ins_pipe(ialu_reg_mem);
13499 %}
13500
13501 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13502 %{
13503 match(Set dst (XorL (AddL src minus_1) src));
13504 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13505 effect(KILL cr);
13506 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13507
13508 format %{ "blsmskq $dst, $src" %}
13509
13510 ins_encode %{
13511 __ blsmskq($dst$$Register, $src$$Register);
13512 %}
13513
13514 ins_pipe(ialu_reg);
13515 %}
13516
13517 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13518 %{
13519 match(Set dst (AndL (AddL src minus_1) src) );
13520 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13521 effect(KILL cr);
13522 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13523
13524 format %{ "blsrq $dst, $src" %}
13525
13526 ins_encode %{
13527 __ blsrq($dst$$Register, $src$$Register);
13528 %}
13529
13530 ins_pipe(ialu_reg);
13531 %}
13532
13533 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13534 %{
13535 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
13536 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13537 effect(KILL cr);
13538 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13539
13540 ins_cost(125);
13541 format %{ "blsrq $dst, $src" %}
13542
13543 ins_encode %{
13544 __ blsrq($dst$$Register, $src$$Address);
13545 %}
13546
13547 ins_pipe(ialu_reg);
13548 %}
13549
13550 // Or Instructions
13551 // Or Register with Register
13552 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13553 %{
13554 predicate(!UseAPX);
13555 match(Set dst (OrL dst src));
13556 effect(KILL cr);
13557 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13558
13559 format %{ "orq $dst, $src\t# long" %}
13560 ins_encode %{
13561 __ orq($dst$$Register, $src$$Register);
13562 %}
13563 ins_pipe(ialu_reg_reg);
13564 %}
13565
13566 // Or Register with Register using New Data Destination (NDD)
13567 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13568 %{
13569 predicate(UseAPX);
13570 match(Set dst (OrL src1 src2));
13571 effect(KILL cr);
13572 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13573
13574 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13575 ins_encode %{
13576 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13577
13578 %}
13579 ins_pipe(ialu_reg_reg);
13580 %}
13581
13582 // Use any_RegP to match R15 (TLS register) without spilling.
13583 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
13584 predicate(!UseAPX);
13585 match(Set dst (OrL dst (CastP2X src)));
13586 effect(KILL cr);
13587 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13588
13589 format %{ "orq $dst, $src\t# long" %}
13590 ins_encode %{
13591 __ orq($dst$$Register, $src$$Register);
13592 %}
13593 ins_pipe(ialu_reg_reg);
13594 %}
13595
13596 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
13597 predicate(UseAPX);
13598 match(Set dst (OrL src1 (CastP2X src2)));
13599 effect(KILL cr);
13600 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13601
13602 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13603 ins_encode %{
13604 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13605 %}
13606 ins_pipe(ialu_reg_reg);
13607 %}
13608
13609 // Or Register with Immediate
13610 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13611 %{
13612 predicate(!UseAPX);
13613 match(Set dst (OrL dst src));
13614 effect(KILL cr);
13615 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13616
13617 format %{ "orq $dst, $src\t# long" %}
13618 ins_encode %{
13619 __ orq($dst$$Register, $src$$constant);
13620 %}
13621 ins_pipe(ialu_reg);
13622 %}
13623
13624 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13625 %{
13626 predicate(UseAPX);
13627 match(Set dst (OrL src1 src2));
13628 effect(KILL cr);
13629 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13630
13631 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13632 ins_encode %{
13633 __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13634 %}
13635 ins_pipe(ialu_reg);
13636 %}
13637
13638 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
13639 %{
13640 predicate(UseAPX);
13641 match(Set dst (OrL src1 src2));
13642 effect(KILL cr);
13643 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13644
13645 format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
13646 ins_encode %{
13647 __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
13648 %}
13649 ins_pipe(ialu_reg);
13650 %}
13651
13652 // Or Register with Memory
13653 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13654 %{
13655 match(Set dst (OrL dst (LoadL src)));
13656 effect(KILL cr);
13657 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13658
13659 ins_cost(150);
13660 format %{ "orq $dst, $src\t# long" %}
13661 ins_encode %{
13662 __ orq($dst$$Register, $src$$Address);
13663 %}
13664 ins_pipe(ialu_reg_mem);
13665 %}
13666
13667 // Or Memory with Register
13668 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13669 %{
13670 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13671 effect(KILL cr);
13672 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13673
13674 ins_cost(150);
13675 format %{ "orq $dst, $src\t# long" %}
13676 ins_encode %{
13677 __ orq($dst$$Address, $src$$Register);
13678 %}
13679 ins_pipe(ialu_mem_reg);
13680 %}
13681
13682 // Or Memory with Immediate
13683 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13684 %{
13685 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13686 effect(KILL cr);
13687 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13688
13689 ins_cost(125);
13690 format %{ "orq $dst, $src\t# long" %}
13691 ins_encode %{
13692 __ orq($dst$$Address, $src$$constant);
13693 %}
13694 ins_pipe(ialu_mem_imm);
13695 %}
13696
13697 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
13698 %{
13699 // con should be a pure 64-bit power of 2 immediate
13700 // because AND/OR works well enough for 8/32-bit values.
13701 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
13702
13703 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
13704 effect(KILL cr);
13705
13706 ins_cost(125);
13707 format %{ "btsq $dst, log2($con)\t# long" %}
13708 ins_encode %{
13709 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
13710 %}
13711 ins_pipe(ialu_mem_imm);
13712 %}
13713
13714 // Xor Instructions
13715 // Xor Register with Register
13716 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13717 %{
13718 predicate(!UseAPX);
13719 match(Set dst (XorL dst src));
13720 effect(KILL cr);
13721 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13722
13723 format %{ "xorq $dst, $src\t# long" %}
13724 ins_encode %{
13725 __ xorq($dst$$Register, $src$$Register);
13726 %}
13727 ins_pipe(ialu_reg_reg);
13728 %}
13729
13730 // Xor Register with Register using New Data Destination (NDD)
13731 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13732 %{
13733 predicate(UseAPX);
13734 match(Set dst (XorL src1 src2));
13735 effect(KILL cr);
13736 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13737
13738 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13739 ins_encode %{
13740 __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13741 %}
13742 ins_pipe(ialu_reg_reg);
13743 %}
13744
13745 // Xor Register with Immediate -1
13746 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
13747 %{
13748 predicate(!UseAPX);
13749 match(Set dst (XorL dst imm));
13750
13751 format %{ "notq $dst" %}
13752 ins_encode %{
13753 __ notq($dst$$Register);
13754 %}
13755 ins_pipe(ialu_reg);
13756 %}
13757
13758 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
13759 %{
13760 predicate(UseAPX);
13761 match(Set dst (XorL src imm));
13762 flag(PD::Flag_ndd_demotable_opr1);
13763
13764 format %{ "enotq $dst, $src" %}
13765 ins_encode %{
13766 __ enotq($dst$$Register, $src$$Register);
13767 %}
13768 ins_pipe(ialu_reg);
13769 %}
13770
13771 // Xor Register with Immediate
13772 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13773 %{
13774 // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
13775 predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13776 match(Set dst (XorL dst src));
13777 effect(KILL cr);
13778 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13779
13780 format %{ "xorq $dst, $src\t# long" %}
13781 ins_encode %{
13782 __ xorq($dst$$Register, $src$$constant);
13783 %}
13784 ins_pipe(ialu_reg);
13785 %}
13786
13787 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13788 %{
13789 // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
13790 predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13791 match(Set dst (XorL src1 src2));
13792 effect(KILL cr);
13793 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13794
13795 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13796 ins_encode %{
13797 __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13798 %}
13799 ins_pipe(ialu_reg);
13800 %}
13801
13802 // Xor Register with Memory
13803 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13804 %{
13805 match(Set dst (XorL dst (LoadL src)));
13806 effect(KILL cr);
13807 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13808
13809 ins_cost(150);
13810 format %{ "xorq $dst, $src\t# long" %}
13811 ins_encode %{
13812 __ xorq($dst$$Register, $src$$Address);
13813 %}
13814 ins_pipe(ialu_reg_mem);
13815 %}
13816
13817 // Xor Memory with Register
13818 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13819 %{
13820 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13821 effect(KILL cr);
13822 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13823
13824 ins_cost(150);
13825 format %{ "xorq $dst, $src\t# long" %}
13826 ins_encode %{
13827 __ xorq($dst$$Address, $src$$Register);
13828 %}
13829 ins_pipe(ialu_mem_reg);
13830 %}
13831
13832 // Xor Memory with Immediate
13833 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13834 %{
13835 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13836 effect(KILL cr);
13837 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13838
13839 ins_cost(125);
13840 format %{ "xorq $dst, $src\t# long" %}
13841 ins_encode %{
13842 __ xorq($dst$$Address, $src$$constant);
13843 %}
13844 ins_pipe(ialu_mem_imm);
13845 %}
13846
13847 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
13848 %{
13849 match(Set dst (CmpLTMask p q));
13850 effect(KILL cr);
13851
13852 ins_cost(400);
13853 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
13854 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
13855 "negl $dst" %}
13856 ins_encode %{
13857 __ cmpl($p$$Register, $q$$Register);
13858 __ setcc(Assembler::less, $dst$$Register);
13859 __ negl($dst$$Register);
13860 %}
13861 ins_pipe(pipe_slow);
13862 %}
13863
13864 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
13865 %{
13866 match(Set dst (CmpLTMask dst zero));
13867 effect(KILL cr);
13868
13869 ins_cost(100);
13870 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
13871 ins_encode %{
13872 __ sarl($dst$$Register, 31);
13873 %}
13874 ins_pipe(ialu_reg);
13875 %}
13876
13877 /* Better to save a register than avoid a branch */
13878 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13879 %{
13880 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
13881 effect(KILL cr);
13882 ins_cost(300);
13883 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
13884 "jge done\n\t"
13885 "addl $p,$y\n"
13886 "done: " %}
13887 ins_encode %{
13888 Register Rp = $p$$Register;
13889 Register Rq = $q$$Register;
13890 Register Ry = $y$$Register;
13891 Label done;
13892 __ subl(Rp, Rq);
13893 __ jccb(Assembler::greaterEqual, done);
13894 __ addl(Rp, Ry);
13895 __ bind(done);
13896 %}
13897 ins_pipe(pipe_cmplt);
13898 %}
13899
13900 /* Better to save a register than avoid a branch */
13901 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13902 %{
13903 match(Set y (AndI (CmpLTMask p q) y));
13904 effect(KILL cr);
13905
13906 ins_cost(300);
13907
13908 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
13909 "jlt done\n\t"
13910 "xorl $y, $y\n"
13911 "done: " %}
13912 ins_encode %{
13913 Register Rp = $p$$Register;
13914 Register Rq = $q$$Register;
13915 Register Ry = $y$$Register;
13916 Label done;
13917 __ cmpl(Rp, Rq);
13918 __ jccb(Assembler::less, done);
13919 __ xorl(Ry, Ry);
13920 __ bind(done);
13921 %}
13922 ins_pipe(pipe_cmplt);
13923 %}
13924
13925
13926 //---------- FP Instructions------------------------------------------------
13927
13928 // Really expensive, avoid
13929 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
13930 %{
13931 match(Set cr (CmpF src1 src2));
13932
13933 ins_cost(500);
13934 format %{ "ucomiss $src1, $src2\n\t"
13935 "jnp,s exit\n\t"
13936 "pushfq\t# saw NaN, set CF\n\t"
13937 "andq [rsp], #0xffffff2b\n\t"
13938 "popfq\n"
13939 "exit:" %}
13940 ins_encode %{
13941 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13942 emit_cmpfp_fixup(masm);
13943 %}
13944 ins_pipe(pipe_slow);
13945 %}
13946
13947 instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
13948 match(Set cr (CmpF src1 src2));
13949
13950 ins_cost(100);
13951 format %{ "ucomiss $src1, $src2" %}
13952 ins_encode %{
13953 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13954 %}
13955 ins_pipe(pipe_slow);
13956 %}
13957
13958 instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
13959 match(Set cr (CmpF src1 src2));
13960
13961 ins_cost(100);
13962 format %{ "evucomxss $src1, $src2" %}
13963 ins_encode %{
13964 __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
13965 %}
13966 ins_pipe(pipe_slow);
13967 %}
13968
13969 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
13970 match(Set cr (CmpF src1 (LoadF src2)));
13971
13972 ins_cost(100);
13973 format %{ "ucomiss $src1, $src2" %}
13974 ins_encode %{
13975 __ ucomiss($src1$$XMMRegister, $src2$$Address);
13976 %}
13977 ins_pipe(pipe_slow);
13978 %}
13979
13980 instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
13981 match(Set cr (CmpF src1 (LoadF src2)));
13982
13983 ins_cost(100);
13984 format %{ "evucomxss $src1, $src2" %}
13985 ins_encode %{
13986 __ evucomxss($src1$$XMMRegister, $src2$$Address);
13987 %}
13988 ins_pipe(pipe_slow);
13989 %}
13990
13991 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
13992 match(Set cr (CmpF src con));
13993
13994 ins_cost(100);
13995 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13996 ins_encode %{
13997 __ ucomiss($src$$XMMRegister, $constantaddress($con));
13998 %}
13999 ins_pipe(pipe_slow);
14000 %}
14001
14002 instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
14003 match(Set cr (CmpF src con));
14004
14005 ins_cost(100);
14006 format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
14007 ins_encode %{
14008 __ evucomxss($src$$XMMRegister, $constantaddress($con));
14009 %}
14010 ins_pipe(pipe_slow);
14011 %}
14012
14013 // Really expensive, avoid
14014 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
14015 %{
14016 match(Set cr (CmpD src1 src2));
14017
14018 ins_cost(500);
14019 format %{ "ucomisd $src1, $src2\n\t"
14020 "jnp,s exit\n\t"
14021 "pushfq\t# saw NaN, set CF\n\t"
14022 "andq [rsp], #0xffffff2b\n\t"
14023 "popfq\n"
14024 "exit:" %}
14025 ins_encode %{
14026 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14027 emit_cmpfp_fixup(masm);
14028 %}
14029 ins_pipe(pipe_slow);
14030 %}
14031
14032 instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
14033 match(Set cr (CmpD src1 src2));
14034
14035 ins_cost(100);
14036 format %{ "ucomisd $src1, $src2 test" %}
14037 ins_encode %{
14038 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14039 %}
14040 ins_pipe(pipe_slow);
14041 %}
14042
14043 instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
14044 match(Set cr (CmpD src1 src2));
14045
14046 ins_cost(100);
14047 format %{ "evucomxsd $src1, $src2 test" %}
14048 ins_encode %{
14049 __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
14050 %}
14051 ins_pipe(pipe_slow);
14052 %}
14053
14054 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
14055 match(Set cr (CmpD src1 (LoadD src2)));
14056
14057 ins_cost(100);
14058 format %{ "ucomisd $src1, $src2" %}
14059 ins_encode %{
14060 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14061 %}
14062 ins_pipe(pipe_slow);
14063 %}
14064
14065 instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
14066 match(Set cr (CmpD src1 (LoadD src2)));
14067
14068 ins_cost(100);
14069 format %{ "evucomxsd $src1, $src2" %}
14070 ins_encode %{
14071 __ evucomxsd($src1$$XMMRegister, $src2$$Address);
14072 %}
14073 ins_pipe(pipe_slow);
14074 %}
14075
14076 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
14077 match(Set cr (CmpD src con));
14078 ins_cost(100);
14079 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14080 ins_encode %{
14081 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14082 %}
14083 ins_pipe(pipe_slow);
14084 %}
14085
14086 instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
14087 match(Set cr (CmpD src con));
14088
14089 ins_cost(100);
14090 format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14091 ins_encode %{
14092 __ evucomxsd($src$$XMMRegister, $constantaddress($con));
14093 %}
14094 ins_pipe(pipe_slow);
14095 %}
14096
14097 // Compare into -1,0,1
14098 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
14099 %{
14100 match(Set dst (CmpF3 src1 src2));
14101 effect(KILL cr);
14102
14103 ins_cost(275);
14104 format %{ "ucomiss $src1, $src2\n\t"
14105 "movl $dst, #-1\n\t"
14106 "jp,s done\n\t"
14107 "jb,s done\n\t"
14108 "setne $dst\n\t"
14109 "movzbl $dst, $dst\n"
14110 "done:" %}
14111 ins_encode %{
14112 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
14113 emit_cmpfp3(masm, $dst$$Register);
14114 %}
14115 ins_pipe(pipe_slow);
14116 %}
14117
14118 // Compare into -1,0,1
14119 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
14120 %{
14121 match(Set dst (CmpF3 src1 (LoadF src2)));
14122 effect(KILL cr);
14123
14124 ins_cost(275);
14125 format %{ "ucomiss $src1, $src2\n\t"
14126 "movl $dst, #-1\n\t"
14127 "jp,s done\n\t"
14128 "jb,s done\n\t"
14129 "setne $dst\n\t"
14130 "movzbl $dst, $dst\n"
14131 "done:" %}
14132 ins_encode %{
14133 __ ucomiss($src1$$XMMRegister, $src2$$Address);
14134 emit_cmpfp3(masm, $dst$$Register);
14135 %}
14136 ins_pipe(pipe_slow);
14137 %}
14138
14139 // Compare into -1,0,1
14140 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
14141 match(Set dst (CmpF3 src con));
14142 effect(KILL cr);
14143
14144 ins_cost(275);
14145 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
14146 "movl $dst, #-1\n\t"
14147 "jp,s done\n\t"
14148 "jb,s done\n\t"
14149 "setne $dst\n\t"
14150 "movzbl $dst, $dst\n"
14151 "done:" %}
14152 ins_encode %{
14153 __ ucomiss($src$$XMMRegister, $constantaddress($con));
14154 emit_cmpfp3(masm, $dst$$Register);
14155 %}
14156 ins_pipe(pipe_slow);
14157 %}
14158
14159 // Compare into -1,0,1
14160 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
14161 %{
14162 match(Set dst (CmpD3 src1 src2));
14163 effect(KILL cr);
14164
14165 ins_cost(275);
14166 format %{ "ucomisd $src1, $src2\n\t"
14167 "movl $dst, #-1\n\t"
14168 "jp,s done\n\t"
14169 "jb,s done\n\t"
14170 "setne $dst\n\t"
14171 "movzbl $dst, $dst\n"
14172 "done:" %}
14173 ins_encode %{
14174 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14175 emit_cmpfp3(masm, $dst$$Register);
14176 %}
14177 ins_pipe(pipe_slow);
14178 %}
14179
14180 // Compare into -1,0,1
14181 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
14182 %{
14183 match(Set dst (CmpD3 src1 (LoadD src2)));
14184 effect(KILL cr);
14185
14186 ins_cost(275);
14187 format %{ "ucomisd $src1, $src2\n\t"
14188 "movl $dst, #-1\n\t"
14189 "jp,s done\n\t"
14190 "jb,s done\n\t"
14191 "setne $dst\n\t"
14192 "movzbl $dst, $dst\n"
14193 "done:" %}
14194 ins_encode %{
14195 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14196 emit_cmpfp3(masm, $dst$$Register);
14197 %}
14198 ins_pipe(pipe_slow);
14199 %}
14200
14201 // Compare into -1,0,1
14202 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
14203 match(Set dst (CmpD3 src con));
14204 effect(KILL cr);
14205
14206 ins_cost(275);
14207 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
14208 "movl $dst, #-1\n\t"
14209 "jp,s done\n\t"
14210 "jb,s done\n\t"
14211 "setne $dst\n\t"
14212 "movzbl $dst, $dst\n"
14213 "done:" %}
14214 ins_encode %{
14215 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14216 emit_cmpfp3(masm, $dst$$Register);
14217 %}
14218 ins_pipe(pipe_slow);
14219 %}
14220
14221 //----------Arithmetic Conversion Instructions---------------------------------
14222
14223 instruct convF2D_reg_reg(regD dst, regF src)
14224 %{
14225 match(Set dst (ConvF2D src));
14226
14227 format %{ "cvtss2sd $dst, $src" %}
14228 ins_encode %{
14229 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
14230 %}
14231 ins_pipe(pipe_slow); // XXX
14232 %}
14233
14234 instruct convF2D_reg_mem(regD dst, memory src)
14235 %{
14236 predicate(UseAVX == 0);
14237 match(Set dst (ConvF2D (LoadF src)));
14238
14239 format %{ "cvtss2sd $dst, $src" %}
14240 ins_encode %{
14241 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
14242 %}
14243 ins_pipe(pipe_slow); // XXX
14244 %}
14245
14246 instruct convD2F_reg_reg(regF dst, regD src)
14247 %{
14248 match(Set dst (ConvD2F src));
14249
14250 format %{ "cvtsd2ss $dst, $src" %}
14251 ins_encode %{
14252 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
14253 %}
14254 ins_pipe(pipe_slow); // XXX
14255 %}
14256
14257 instruct convD2F_reg_mem(regF dst, memory src)
14258 %{
14259 predicate(UseAVX == 0);
14260 match(Set dst (ConvD2F (LoadD src)));
14261
14262 format %{ "cvtsd2ss $dst, $src" %}
14263 ins_encode %{
14264 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
14265 %}
14266 ins_pipe(pipe_slow); // XXX
14267 %}
14268
14269 // XXX do mem variants
14270 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
14271 %{
14272 predicate(!VM_Version::supports_avx10_2());
14273 match(Set dst (ConvF2I src));
14274 effect(KILL cr);
14275 format %{ "convert_f2i $dst, $src" %}
14276 ins_encode %{
14277 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14278 %}
14279 ins_pipe(pipe_slow);
14280 %}
14281
14282 instruct convF2I_reg_reg_avx10_2(rRegI dst, regF src)
14283 %{
14284 predicate(VM_Version::supports_avx10_2());
14285 match(Set dst (ConvF2I src));
14286 format %{ "evcvttss2sisl $dst, $src" %}
14287 ins_encode %{
14288 __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
14289 %}
14290 ins_pipe(pipe_slow);
14291 %}
14292
14293 instruct convF2I_reg_mem_avx10_2(rRegI dst, memory src)
14294 %{
14295 predicate(VM_Version::supports_avx10_2());
14296 match(Set dst (ConvF2I (LoadF src)));
14297 format %{ "evcvttss2sisl $dst, $src" %}
14298 ins_encode %{
14299 __ evcvttss2sisl($dst$$Register, $src$$Address);
14300 %}
14301 ins_pipe(pipe_slow);
14302 %}
14303
14304 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
14305 %{
14306 predicate(!VM_Version::supports_avx10_2());
14307 match(Set dst (ConvF2L src));
14308 effect(KILL cr);
14309 format %{ "convert_f2l $dst, $src"%}
14310 ins_encode %{
14311 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14312 %}
14313 ins_pipe(pipe_slow);
14314 %}
14315
14316 instruct convF2L_reg_reg_avx10_2(rRegL dst, regF src)
14317 %{
14318 predicate(VM_Version::supports_avx10_2());
14319 match(Set dst (ConvF2L src));
14320 format %{ "evcvttss2sisq $dst, $src" %}
14321 ins_encode %{
14322 __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
14323 %}
14324 ins_pipe(pipe_slow);
14325 %}
14326
14327 instruct convF2L_reg_mem_avx10_2(rRegL dst, memory src)
14328 %{
14329 predicate(VM_Version::supports_avx10_2());
14330 match(Set dst (ConvF2L (LoadF src)));
14331 format %{ "evcvttss2sisq $dst, $src" %}
14332 ins_encode %{
14333 __ evcvttss2sisq($dst$$Register, $src$$Address);
14334 %}
14335 ins_pipe(pipe_slow);
14336 %}
14337
14338 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
14339 %{
14340 predicate(!VM_Version::supports_avx10_2());
14341 match(Set dst (ConvD2I src));
14342 effect(KILL cr);
14343 format %{ "convert_d2i $dst, $src"%}
14344 ins_encode %{
14345 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14346 %}
14347 ins_pipe(pipe_slow);
14348 %}
14349
14350 instruct convD2I_reg_reg_avx10_2(rRegI dst, regD src)
14351 %{
14352 predicate(VM_Version::supports_avx10_2());
14353 match(Set dst (ConvD2I src));
14354 format %{ "evcvttsd2sisl $dst, $src" %}
14355 ins_encode %{
14356 __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
14357 %}
14358 ins_pipe(pipe_slow);
14359 %}
14360
14361 instruct convD2I_reg_mem_avx10_2(rRegI dst, memory src)
14362 %{
14363 predicate(VM_Version::supports_avx10_2());
14364 match(Set dst (ConvD2I (LoadD src)));
14365 format %{ "evcvttsd2sisl $dst, $src" %}
14366 ins_encode %{
14367 __ evcvttsd2sisl($dst$$Register, $src$$Address);
14368 %}
14369 ins_pipe(pipe_slow);
14370 %}
14371
14372 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
14373 %{
14374 predicate(!VM_Version::supports_avx10_2());
14375 match(Set dst (ConvD2L src));
14376 effect(KILL cr);
14377 format %{ "convert_d2l $dst, $src"%}
14378 ins_encode %{
14379 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14380 %}
14381 ins_pipe(pipe_slow);
14382 %}
14383
14384 instruct convD2L_reg_reg_avx10_2(rRegL dst, regD src)
14385 %{
14386 predicate(VM_Version::supports_avx10_2());
14387 match(Set dst (ConvD2L src));
14388 format %{ "evcvttsd2sisq $dst, $src" %}
14389 ins_encode %{
14390 __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
14391 %}
14392 ins_pipe(pipe_slow);
14393 %}
14394
14395 instruct convD2L_reg_mem_avx10_2(rRegL dst, memory src)
14396 %{
14397 predicate(VM_Version::supports_avx10_2());
14398 match(Set dst (ConvD2L (LoadD src)));
14399 format %{ "evcvttsd2sisq $dst, $src" %}
14400 ins_encode %{
14401 __ evcvttsd2sisq($dst$$Register, $src$$Address);
14402 %}
14403 ins_pipe(pipe_slow);
14404 %}
14405
14406 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14407 %{
14408 match(Set dst (RoundD src));
14409 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14410 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
14411 ins_encode %{
14412 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14413 %}
14414 ins_pipe(pipe_slow);
14415 %}
14416
14417 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14418 %{
14419 match(Set dst (RoundF src));
14420 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14421 format %{ "round_float $dst,$src" %}
14422 ins_encode %{
14423 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14424 %}
14425 ins_pipe(pipe_slow);
14426 %}
14427
14428 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
14429 %{
14430 predicate(!UseXmmI2F);
14431 match(Set dst (ConvI2F src));
14432
14433 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14434 ins_encode %{
14435 if (UseAVX > 0) {
14436 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14437 }
14438 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
14439 %}
14440 ins_pipe(pipe_slow); // XXX
14441 %}
14442
14443 instruct convI2F_reg_mem(regF dst, memory src)
14444 %{
14445 predicate(UseAVX == 0);
14446 match(Set dst (ConvI2F (LoadI src)));
14447
14448 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14449 ins_encode %{
14450 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
14451 %}
14452 ins_pipe(pipe_slow); // XXX
14453 %}
14454
14455 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
14456 %{
14457 predicate(!UseXmmI2D);
14458 match(Set dst (ConvI2D src));
14459
14460 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14461 ins_encode %{
14462 if (UseAVX > 0) {
14463 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14464 }
14465 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
14466 %}
14467 ins_pipe(pipe_slow); // XXX
14468 %}
14469
14470 instruct convI2D_reg_mem(regD dst, memory src)
14471 %{
14472 predicate(UseAVX == 0);
14473 match(Set dst (ConvI2D (LoadI src)));
14474
14475 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14476 ins_encode %{
14477 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
14478 %}
14479 ins_pipe(pipe_slow); // XXX
14480 %}
14481
14482 instruct convXI2F_reg(regF dst, rRegI src)
14483 %{
14484 predicate(UseXmmI2F);
14485 match(Set dst (ConvI2F src));
14486
14487 format %{ "movdl $dst, $src\n\t"
14488 "cvtdq2psl $dst, $dst\t# i2f" %}
14489 ins_encode %{
14490 __ movdl($dst$$XMMRegister, $src$$Register);
14491 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
14492 %}
14493 ins_pipe(pipe_slow); // XXX
14494 %}
14495
14496 instruct convXI2D_reg(regD dst, rRegI src)
14497 %{
14498 predicate(UseXmmI2D);
14499 match(Set dst (ConvI2D src));
14500
14501 format %{ "movdl $dst, $src\n\t"
14502 "cvtdq2pdl $dst, $dst\t# i2d" %}
14503 ins_encode %{
14504 __ movdl($dst$$XMMRegister, $src$$Register);
14505 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
14506 %}
14507 ins_pipe(pipe_slow); // XXX
14508 %}
14509
14510 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
14511 %{
14512 match(Set dst (ConvL2F src));
14513
14514 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14515 ins_encode %{
14516 if (UseAVX > 0) {
14517 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14518 }
14519 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
14520 %}
14521 ins_pipe(pipe_slow); // XXX
14522 %}
14523
14524 instruct convL2F_reg_mem(regF dst, memory src)
14525 %{
14526 predicate(UseAVX == 0);
14527 match(Set dst (ConvL2F (LoadL src)));
14528
14529 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14530 ins_encode %{
14531 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
14532 %}
14533 ins_pipe(pipe_slow); // XXX
14534 %}
14535
14536 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
14537 %{
14538 match(Set dst (ConvL2D src));
14539
14540 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14541 ins_encode %{
14542 if (UseAVX > 0) {
14543 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14544 }
14545 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
14546 %}
14547 ins_pipe(pipe_slow); // XXX
14548 %}
14549
14550 instruct convL2D_reg_mem(regD dst, memory src)
14551 %{
14552 predicate(UseAVX == 0);
14553 match(Set dst (ConvL2D (LoadL src)));
14554
14555 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14556 ins_encode %{
14557 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
14558 %}
14559 ins_pipe(pipe_slow); // XXX
14560 %}
14561
14562 instruct convI2L_reg_reg(rRegL dst, rRegI src)
14563 %{
14564 match(Set dst (ConvI2L src));
14565
14566 ins_cost(125);
14567 format %{ "movslq $dst, $src\t# i2l" %}
14568 ins_encode %{
14569 __ movslq($dst$$Register, $src$$Register);
14570 %}
14571 ins_pipe(ialu_reg_reg);
14572 %}
14573
14574 // Zero-extend convert int to long
14575 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
14576 %{
14577 match(Set dst (AndL (ConvI2L src) mask));
14578
14579 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14580 ins_encode %{
14581 if ($dst$$reg != $src$$reg) {
14582 __ movl($dst$$Register, $src$$Register);
14583 }
14584 %}
14585 ins_pipe(ialu_reg_reg);
14586 %}
14587
14588 // Zero-extend convert int to long
14589 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
14590 %{
14591 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
14592
14593 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14594 ins_encode %{
14595 __ movl($dst$$Register, $src$$Address);
14596 %}
14597 ins_pipe(ialu_reg_mem);
14598 %}
14599
14600 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
14601 %{
14602 match(Set dst (AndL src mask));
14603
14604 format %{ "movl $dst, $src\t# zero-extend long" %}
14605 ins_encode %{
14606 __ movl($dst$$Register, $src$$Register);
14607 %}
14608 ins_pipe(ialu_reg_reg);
14609 %}
14610
14611 instruct convL2I_reg_reg(rRegI dst, rRegL src)
14612 %{
14613 match(Set dst (ConvL2I src));
14614
14615 format %{ "movl $dst, $src\t# l2i" %}
14616 ins_encode %{
14617 __ movl($dst$$Register, $src$$Register);
14618 %}
14619 ins_pipe(ialu_reg_reg);
14620 %}
14621
14622
14623 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
14624 match(Set dst (MoveF2I src));
14625 effect(DEF dst, USE src);
14626
14627 ins_cost(125);
14628 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
14629 ins_encode %{
14630 __ movl($dst$$Register, Address(rsp, $src$$disp));
14631 %}
14632 ins_pipe(ialu_reg_mem);
14633 %}
14634
14635 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
14636 match(Set dst (MoveI2F src));
14637 effect(DEF dst, USE src);
14638
14639 ins_cost(125);
14640 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
14641 ins_encode %{
14642 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
14643 %}
14644 ins_pipe(pipe_slow);
14645 %}
14646
14647 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
14648 match(Set dst (MoveD2L src));
14649 effect(DEF dst, USE src);
14650
14651 ins_cost(125);
14652 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
14653 ins_encode %{
14654 __ movq($dst$$Register, Address(rsp, $src$$disp));
14655 %}
14656 ins_pipe(ialu_reg_mem);
14657 %}
14658
14659 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
14660 predicate(!UseXmmLoadAndClearUpper);
14661 match(Set dst (MoveL2D src));
14662 effect(DEF dst, USE src);
14663
14664 ins_cost(125);
14665 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
14666 ins_encode %{
14667 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14668 %}
14669 ins_pipe(pipe_slow);
14670 %}
14671
14672 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
14673 predicate(UseXmmLoadAndClearUpper);
14674 match(Set dst (MoveL2D src));
14675 effect(DEF dst, USE src);
14676
14677 ins_cost(125);
14678 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
14679 ins_encode %{
14680 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14681 %}
14682 ins_pipe(pipe_slow);
14683 %}
14684
14685
14686 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
14687 match(Set dst (MoveF2I src));
14688 effect(DEF dst, USE src);
14689
14690 ins_cost(95); // XXX
14691 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
14692 ins_encode %{
14693 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
14694 %}
14695 ins_pipe(pipe_slow);
14696 %}
14697
14698 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
14699 match(Set dst (MoveI2F src));
14700 effect(DEF dst, USE src);
14701
14702 ins_cost(100);
14703 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
14704 ins_encode %{
14705 __ movl(Address(rsp, $dst$$disp), $src$$Register);
14706 %}
14707 ins_pipe( ialu_mem_reg );
14708 %}
14709
14710 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
14711 match(Set dst (MoveD2L src));
14712 effect(DEF dst, USE src);
14713
14714 ins_cost(95); // XXX
14715 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
14716 ins_encode %{
14717 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
14718 %}
14719 ins_pipe(pipe_slow);
14720 %}
14721
14722 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
14723 match(Set dst (MoveL2D src));
14724 effect(DEF dst, USE src);
14725
14726 ins_cost(100);
14727 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
14728 ins_encode %{
14729 __ movq(Address(rsp, $dst$$disp), $src$$Register);
14730 %}
14731 ins_pipe(ialu_mem_reg);
14732 %}
14733
14734 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
14735 match(Set dst (MoveF2I src));
14736 effect(DEF dst, USE src);
14737 ins_cost(85);
14738 format %{ "movd $dst,$src\t# MoveF2I" %}
14739 ins_encode %{
14740 __ movdl($dst$$Register, $src$$XMMRegister);
14741 %}
14742 ins_pipe( pipe_slow );
14743 %}
14744
14745 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
14746 match(Set dst (MoveD2L src));
14747 effect(DEF dst, USE src);
14748 ins_cost(85);
14749 format %{ "movd $dst,$src\t# MoveD2L" %}
14750 ins_encode %{
14751 __ movdq($dst$$Register, $src$$XMMRegister);
14752 %}
14753 ins_pipe( pipe_slow );
14754 %}
14755
14756 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
14757 match(Set dst (MoveI2F src));
14758 effect(DEF dst, USE src);
14759 ins_cost(100);
14760 format %{ "movd $dst,$src\t# MoveI2F" %}
14761 ins_encode %{
14762 __ movdl($dst$$XMMRegister, $src$$Register);
14763 %}
14764 ins_pipe( pipe_slow );
14765 %}
14766
14767 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14768 match(Set dst (MoveL2D src));
14769 effect(DEF dst, USE src);
14770 ins_cost(100);
14771 format %{ "movd $dst,$src\t# MoveL2D" %}
14772 ins_encode %{
14773 __ movdq($dst$$XMMRegister, $src$$Register);
14774 %}
14775 ins_pipe( pipe_slow );
14776 %}
14777
14778
14779 // Fast clearing of an array
14780 // Small non-constant lenght ClearArray for non-AVX512 targets.
14781 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14782 Universe dummy, rFlagsReg cr)
14783 %{
14784 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14785 match(Set dummy (ClearArray (Binary cnt base) val));
14786 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14787
14788 format %{ $$template
14789 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14790 $$emit$$"jg LARGE\n\t"
14791 $$emit$$"dec rcx\n\t"
14792 $$emit$$"js DONE\t# Zero length\n\t"
14793 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14794 $$emit$$"dec rcx\n\t"
14795 $$emit$$"jge LOOP\n\t"
14796 $$emit$$"jmp DONE\n\t"
14797 $$emit$$"# LARGE:\n\t"
14798 if (UseFastStosb) {
14799 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14800 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14801 } else if (UseXMMForObjInit) {
14802 $$emit$$"movdq $tmp, $val\n\t"
14803 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14804 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14805 $$emit$$"jmpq L_zero_64_bytes\n\t"
14806 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14807 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14808 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14809 $$emit$$"add 0x40,rax\n\t"
14810 $$emit$$"# L_zero_64_bytes:\n\t"
14811 $$emit$$"sub 0x8,rcx\n\t"
14812 $$emit$$"jge L_loop\n\t"
14813 $$emit$$"add 0x4,rcx\n\t"
14814 $$emit$$"jl L_tail\n\t"
14815 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14816 $$emit$$"add 0x20,rax\n\t"
14817 $$emit$$"sub 0x4,rcx\n\t"
14818 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14819 $$emit$$"add 0x4,rcx\n\t"
14820 $$emit$$"jle L_end\n\t"
14821 $$emit$$"dec rcx\n\t"
14822 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14823 $$emit$$"vmovq xmm0,(rax)\n\t"
14824 $$emit$$"add 0x8,rax\n\t"
14825 $$emit$$"dec rcx\n\t"
14826 $$emit$$"jge L_sloop\n\t"
14827 $$emit$$"# L_end:\n\t"
14828 } else {
14829 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14830 }
14831 $$emit$$"# DONE"
14832 %}
14833 ins_encode %{
14834 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14835 $tmp$$XMMRegister, false, false);
14836 %}
14837 ins_pipe(pipe_slow);
14838 %}
14839
14840 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14841 Universe dummy, rFlagsReg cr)
14842 %{
14843 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14844 match(Set dummy (ClearArray (Binary cnt base) val));
14845 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14846
14847 format %{ $$template
14848 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14849 $$emit$$"jg LARGE\n\t"
14850 $$emit$$"dec rcx\n\t"
14851 $$emit$$"js DONE\t# Zero length\n\t"
14852 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14853 $$emit$$"dec rcx\n\t"
14854 $$emit$$"jge LOOP\n\t"
14855 $$emit$$"jmp DONE\n\t"
14856 $$emit$$"# LARGE:\n\t"
14857 if (UseXMMForObjInit) {
14858 $$emit$$"movdq $tmp, $val\n\t"
14859 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14860 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14861 $$emit$$"jmpq L_zero_64_bytes\n\t"
14862 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14863 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14864 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14865 $$emit$$"add 0x40,rax\n\t"
14866 $$emit$$"# L_zero_64_bytes:\n\t"
14867 $$emit$$"sub 0x8,rcx\n\t"
14868 $$emit$$"jge L_loop\n\t"
14869 $$emit$$"add 0x4,rcx\n\t"
14870 $$emit$$"jl L_tail\n\t"
14871 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14872 $$emit$$"add 0x20,rax\n\t"
14873 $$emit$$"sub 0x4,rcx\n\t"
14874 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14875 $$emit$$"add 0x4,rcx\n\t"
14876 $$emit$$"jle L_end\n\t"
14877 $$emit$$"dec rcx\n\t"
14878 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14879 $$emit$$"vmovq xmm0,(rax)\n\t"
14880 $$emit$$"add 0x8,rax\n\t"
14881 $$emit$$"dec rcx\n\t"
14882 $$emit$$"jge L_sloop\n\t"
14883 $$emit$$"# L_end:\n\t"
14884 } else {
14885 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14886 }
14887 $$emit$$"# DONE"
14888 %}
14889 ins_encode %{
14890 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14891 $tmp$$XMMRegister, false, true);
14892 %}
14893 ins_pipe(pipe_slow);
14894 %}
14895
14896 // Small non-constant length ClearArray for AVX512 targets.
14897 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14898 Universe dummy, rFlagsReg cr)
14899 %{
14900 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14901 match(Set dummy (ClearArray (Binary cnt base) val));
14902 ins_cost(125);
14903 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14904
14905 format %{ $$template
14906 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14907 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14908 $$emit$$"jg LARGE\n\t"
14909 $$emit$$"dec rcx\n\t"
14910 $$emit$$"js DONE\t# Zero length\n\t"
14911 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14912 $$emit$$"dec rcx\n\t"
14913 $$emit$$"jge LOOP\n\t"
14914 $$emit$$"jmp DONE\n\t"
14915 $$emit$$"# LARGE:\n\t"
14916 if (UseFastStosb) {
14917 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14918 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14919 } else if (UseXMMForObjInit) {
14920 $$emit$$"mov rdi,rax\n\t"
14921 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14922 $$emit$$"jmpq L_zero_64_bytes\n\t"
14923 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14924 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14925 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14926 $$emit$$"add 0x40,rax\n\t"
14927 $$emit$$"# L_zero_64_bytes:\n\t"
14928 $$emit$$"sub 0x8,rcx\n\t"
14929 $$emit$$"jge L_loop\n\t"
14930 $$emit$$"add 0x4,rcx\n\t"
14931 $$emit$$"jl L_tail\n\t"
14932 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14933 $$emit$$"add 0x20,rax\n\t"
14934 $$emit$$"sub 0x4,rcx\n\t"
14935 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14936 $$emit$$"add 0x4,rcx\n\t"
14937 $$emit$$"jle L_end\n\t"
14938 $$emit$$"dec rcx\n\t"
14939 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14940 $$emit$$"vmovq xmm0,(rax)\n\t"
14941 $$emit$$"add 0x8,rax\n\t"
14942 $$emit$$"dec rcx\n\t"
14943 $$emit$$"jge L_sloop\n\t"
14944 $$emit$$"# L_end:\n\t"
14945 } else {
14946 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14947 }
14948 $$emit$$"# DONE"
14949 %}
14950 ins_encode %{
14951 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14952 $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
14953 %}
14954 ins_pipe(pipe_slow);
14955 %}
14956
14957 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14958 Universe dummy, rFlagsReg cr)
14959 %{
14960 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14961 match(Set dummy (ClearArray (Binary cnt base) val));
14962 ins_cost(125);
14963 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14964
14965 format %{ $$template
14966 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14967 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14968 $$emit$$"jg LARGE\n\t"
14969 $$emit$$"dec rcx\n\t"
14970 $$emit$$"js DONE\t# Zero length\n\t"
14971 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14972 $$emit$$"dec rcx\n\t"
14973 $$emit$$"jge LOOP\n\t"
14974 $$emit$$"jmp DONE\n\t"
14975 $$emit$$"# LARGE:\n\t"
14976 if (UseFastStosb) {
14977 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14978 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14979 } else if (UseXMMForObjInit) {
14980 $$emit$$"mov rdi,rax\n\t"
14981 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14982 $$emit$$"jmpq L_zero_64_bytes\n\t"
14983 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14984 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14985 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14986 $$emit$$"add 0x40,rax\n\t"
14987 $$emit$$"# L_zero_64_bytes:\n\t"
14988 $$emit$$"sub 0x8,rcx\n\t"
14989 $$emit$$"jge L_loop\n\t"
14990 $$emit$$"add 0x4,rcx\n\t"
14991 $$emit$$"jl L_tail\n\t"
14992 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14993 $$emit$$"add 0x20,rax\n\t"
14994 $$emit$$"sub 0x4,rcx\n\t"
14995 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14996 $$emit$$"add 0x4,rcx\n\t"
14997 $$emit$$"jle L_end\n\t"
14998 $$emit$$"dec rcx\n\t"
14999 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15000 $$emit$$"vmovq xmm0,(rax)\n\t"
15001 $$emit$$"add 0x8,rax\n\t"
15002 $$emit$$"dec rcx\n\t"
15003 $$emit$$"jge L_sloop\n\t"
15004 $$emit$$"# L_end:\n\t"
15005 } else {
15006 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
15007 }
15008 $$emit$$"# DONE"
15009 %}
15010 ins_encode %{
15011 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15012 $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
15013 %}
15014 ins_pipe(pipe_slow);
15015 %}
15016
15017 // Large non-constant length ClearArray for non-AVX512 targets.
15018 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15019 Universe dummy, rFlagsReg cr)
15020 %{
15021 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15022 match(Set dummy (ClearArray (Binary cnt base) val));
15023 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15024
15025 format %{ $$template
15026 if (UseFastStosb) {
15027 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15028 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15029 } else if (UseXMMForObjInit) {
15030 $$emit$$"movdq $tmp, $val\n\t"
15031 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15032 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15033 $$emit$$"jmpq L_zero_64_bytes\n\t"
15034 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15035 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15036 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15037 $$emit$$"add 0x40,rax\n\t"
15038 $$emit$$"# L_zero_64_bytes:\n\t"
15039 $$emit$$"sub 0x8,rcx\n\t"
15040 $$emit$$"jge L_loop\n\t"
15041 $$emit$$"add 0x4,rcx\n\t"
15042 $$emit$$"jl L_tail\n\t"
15043 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15044 $$emit$$"add 0x20,rax\n\t"
15045 $$emit$$"sub 0x4,rcx\n\t"
15046 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15047 $$emit$$"add 0x4,rcx\n\t"
15048 $$emit$$"jle L_end\n\t"
15049 $$emit$$"dec rcx\n\t"
15050 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15051 $$emit$$"vmovq xmm0,(rax)\n\t"
15052 $$emit$$"add 0x8,rax\n\t"
15053 $$emit$$"dec rcx\n\t"
15054 $$emit$$"jge L_sloop\n\t"
15055 $$emit$$"# L_end:\n\t"
15056 } else {
15057 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15058 }
15059 %}
15060 ins_encode %{
15061 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15062 $tmp$$XMMRegister, true, false);
15063 %}
15064 ins_pipe(pipe_slow);
15065 %}
15066
15067 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15068 Universe dummy, rFlagsReg cr)
15069 %{
15070 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15071 match(Set dummy (ClearArray (Binary cnt base) val));
15072 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15073
15074 format %{ $$template
15075 if (UseXMMForObjInit) {
15076 $$emit$$"movdq $tmp, $val\n\t"
15077 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15078 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15079 $$emit$$"jmpq L_zero_64_bytes\n\t"
15080 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15081 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15082 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15083 $$emit$$"add 0x40,rax\n\t"
15084 $$emit$$"# L_zero_64_bytes:\n\t"
15085 $$emit$$"sub 0x8,rcx\n\t"
15086 $$emit$$"jge L_loop\n\t"
15087 $$emit$$"add 0x4,rcx\n\t"
15088 $$emit$$"jl L_tail\n\t"
15089 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15090 $$emit$$"add 0x20,rax\n\t"
15091 $$emit$$"sub 0x4,rcx\n\t"
15092 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15093 $$emit$$"add 0x4,rcx\n\t"
15094 $$emit$$"jle L_end\n\t"
15095 $$emit$$"dec rcx\n\t"
15096 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15097 $$emit$$"vmovq xmm0,(rax)\n\t"
15098 $$emit$$"add 0x8,rax\n\t"
15099 $$emit$$"dec rcx\n\t"
15100 $$emit$$"jge L_sloop\n\t"
15101 $$emit$$"# L_end:\n\t"
15102 } else {
15103 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15104 }
15105 %}
15106 ins_encode %{
15107 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15108 $tmp$$XMMRegister, true, true);
15109 %}
15110 ins_pipe(pipe_slow);
15111 %}
15112
15113 // Large non-constant length ClearArray for AVX512 targets.
15114 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15115 Universe dummy, rFlagsReg cr)
15116 %{
15117 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15118 match(Set dummy (ClearArray (Binary cnt base) val));
15119 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15120
15121 format %{ $$template
15122 if (UseFastStosb) {
15123 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15124 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15125 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15126 } else if (UseXMMForObjInit) {
15127 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15128 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15129 $$emit$$"jmpq L_zero_64_bytes\n\t"
15130 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15131 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15132 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15133 $$emit$$"add 0x40,rax\n\t"
15134 $$emit$$"# L_zero_64_bytes:\n\t"
15135 $$emit$$"sub 0x8,rcx\n\t"
15136 $$emit$$"jge L_loop\n\t"
15137 $$emit$$"add 0x4,rcx\n\t"
15138 $$emit$$"jl L_tail\n\t"
15139 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15140 $$emit$$"add 0x20,rax\n\t"
15141 $$emit$$"sub 0x4,rcx\n\t"
15142 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15143 $$emit$$"add 0x4,rcx\n\t"
15144 $$emit$$"jle L_end\n\t"
15145 $$emit$$"dec rcx\n\t"
15146 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15147 $$emit$$"vmovq xmm0,(rax)\n\t"
15148 $$emit$$"add 0x8,rax\n\t"
15149 $$emit$$"dec rcx\n\t"
15150 $$emit$$"jge L_sloop\n\t"
15151 $$emit$$"# L_end:\n\t"
15152 } else {
15153 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15154 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15155 }
15156 %}
15157 ins_encode %{
15158 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15159 $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
15160 %}
15161 ins_pipe(pipe_slow);
15162 %}
15163
15164 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15165 Universe dummy, rFlagsReg cr)
15166 %{
15167 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15168 match(Set dummy (ClearArray (Binary cnt base) val));
15169 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15170
15171 format %{ $$template
15172 if (UseFastStosb) {
15173 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15174 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15175 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15176 } else if (UseXMMForObjInit) {
15177 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15178 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15179 $$emit$$"jmpq L_zero_64_bytes\n\t"
15180 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15181 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15182 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15183 $$emit$$"add 0x40,rax\n\t"
15184 $$emit$$"# L_zero_64_bytes:\n\t"
15185 $$emit$$"sub 0x8,rcx\n\t"
15186 $$emit$$"jge L_loop\n\t"
15187 $$emit$$"add 0x4,rcx\n\t"
15188 $$emit$$"jl L_tail\n\t"
15189 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15190 $$emit$$"add 0x20,rax\n\t"
15191 $$emit$$"sub 0x4,rcx\n\t"
15192 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15193 $$emit$$"add 0x4,rcx\n\t"
15194 $$emit$$"jle L_end\n\t"
15195 $$emit$$"dec rcx\n\t"
15196 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15197 $$emit$$"vmovq xmm0,(rax)\n\t"
15198 $$emit$$"add 0x8,rax\n\t"
15199 $$emit$$"dec rcx\n\t"
15200 $$emit$$"jge L_sloop\n\t"
15201 $$emit$$"# L_end:\n\t"
15202 } else {
15203 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15204 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15205 }
15206 %}
15207 ins_encode %{
15208 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15209 $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
15210 %}
15211 ins_pipe(pipe_slow);
15212 %}
15213
15214 // Small constant length ClearArray for AVX512 targets.
15215 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
15216 %{
15217 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
15218 ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
15219 match(Set dummy (ClearArray (Binary cnt base) val));
15220 ins_cost(100);
15221 effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
15222 format %{ "clear_mem_imm $base , $cnt \n\t" %}
15223 ins_encode %{
15224 __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
15225 %}
15226 ins_pipe(pipe_slow);
15227 %}
15228
15229 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15230 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15231 %{
15232 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15233 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15234 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15235
15236 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15237 ins_encode %{
15238 __ string_compare($str1$$Register, $str2$$Register,
15239 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15240 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
15241 %}
15242 ins_pipe( pipe_slow );
15243 %}
15244
15245 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15246 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15247 %{
15248 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15249 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15250 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15251
15252 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15253 ins_encode %{
15254 __ string_compare($str1$$Register, $str2$$Register,
15255 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15256 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
15257 %}
15258 ins_pipe( pipe_slow );
15259 %}
15260
15261 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15262 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15263 %{
15264 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15265 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15266 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15267
15268 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15269 ins_encode %{
15270 __ string_compare($str1$$Register, $str2$$Register,
15271 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15272 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
15273 %}
15274 ins_pipe( pipe_slow );
15275 %}
15276
15277 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15278 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15279 %{
15280 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15281 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15282 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15283
15284 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15285 ins_encode %{
15286 __ string_compare($str1$$Register, $str2$$Register,
15287 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15288 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
15289 %}
15290 ins_pipe( pipe_slow );
15291 %}
15292
15293 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15294 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15295 %{
15296 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15297 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15298 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15299
15300 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15301 ins_encode %{
15302 __ string_compare($str1$$Register, $str2$$Register,
15303 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15304 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
15305 %}
15306 ins_pipe( pipe_slow );
15307 %}
15308
15309 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15310 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15311 %{
15312 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15313 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15314 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15315
15316 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15317 ins_encode %{
15318 __ string_compare($str1$$Register, $str2$$Register,
15319 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15320 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
15321 %}
15322 ins_pipe( pipe_slow );
15323 %}
15324
15325 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15326 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15327 %{
15328 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15329 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15330 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15331
15332 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15333 ins_encode %{
15334 __ string_compare($str2$$Register, $str1$$Register,
15335 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15336 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
15337 %}
15338 ins_pipe( pipe_slow );
15339 %}
15340
15341 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15342 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15343 %{
15344 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15345 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15346 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15347
15348 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15349 ins_encode %{
15350 __ string_compare($str2$$Register, $str1$$Register,
15351 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15352 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
15353 %}
15354 ins_pipe( pipe_slow );
15355 %}
15356
15357 // fast search of substring with known size.
15358 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15359 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15360 %{
15361 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15362 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15363 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15364
15365 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15366 ins_encode %{
15367 int icnt2 = (int)$int_cnt2$$constant;
15368 if (icnt2 >= 16) {
15369 // IndexOf for constant substrings with size >= 16 elements
15370 // which don't need to be loaded through stack.
15371 __ string_indexofC8($str1$$Register, $str2$$Register,
15372 $cnt1$$Register, $cnt2$$Register,
15373 icnt2, $result$$Register,
15374 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15375 } else {
15376 // Small strings are loaded through stack if they cross page boundary.
15377 __ string_indexof($str1$$Register, $str2$$Register,
15378 $cnt1$$Register, $cnt2$$Register,
15379 icnt2, $result$$Register,
15380 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15381 }
15382 %}
15383 ins_pipe( pipe_slow );
15384 %}
15385
15386 // fast search of substring with known size.
15387 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15388 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15389 %{
15390 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15391 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15392 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15393
15394 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15395 ins_encode %{
15396 int icnt2 = (int)$int_cnt2$$constant;
15397 if (icnt2 >= 8) {
15398 // IndexOf for constant substrings with size >= 8 elements
15399 // which don't need to be loaded through stack.
15400 __ string_indexofC8($str1$$Register, $str2$$Register,
15401 $cnt1$$Register, $cnt2$$Register,
15402 icnt2, $result$$Register,
15403 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15404 } else {
15405 // Small strings are loaded through stack if they cross page boundary.
15406 __ string_indexof($str1$$Register, $str2$$Register,
15407 $cnt1$$Register, $cnt2$$Register,
15408 icnt2, $result$$Register,
15409 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15410 }
15411 %}
15412 ins_pipe( pipe_slow );
15413 %}
15414
15415 // fast search of substring with known size.
15416 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15417 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15418 %{
15419 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15420 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15421 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15422
15423 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15424 ins_encode %{
15425 int icnt2 = (int)$int_cnt2$$constant;
15426 if (icnt2 >= 8) {
15427 // IndexOf for constant substrings with size >= 8 elements
15428 // which don't need to be loaded through stack.
15429 __ string_indexofC8($str1$$Register, $str2$$Register,
15430 $cnt1$$Register, $cnt2$$Register,
15431 icnt2, $result$$Register,
15432 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15433 } else {
15434 // Small strings are loaded through stack if they cross page boundary.
15435 __ string_indexof($str1$$Register, $str2$$Register,
15436 $cnt1$$Register, $cnt2$$Register,
15437 icnt2, $result$$Register,
15438 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15439 }
15440 %}
15441 ins_pipe( pipe_slow );
15442 %}
15443
15444 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15445 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15446 %{
15447 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15448 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15449 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15450
15451 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15452 ins_encode %{
15453 __ string_indexof($str1$$Register, $str2$$Register,
15454 $cnt1$$Register, $cnt2$$Register,
15455 (-1), $result$$Register,
15456 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15457 %}
15458 ins_pipe( pipe_slow );
15459 %}
15460
15461 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15462 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15463 %{
15464 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15465 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15466 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15467
15468 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15469 ins_encode %{
15470 __ string_indexof($str1$$Register, $str2$$Register,
15471 $cnt1$$Register, $cnt2$$Register,
15472 (-1), $result$$Register,
15473 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15474 %}
15475 ins_pipe( pipe_slow );
15476 %}
15477
15478 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15479 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15480 %{
15481 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15482 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15483 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15484
15485 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15486 ins_encode %{
15487 __ string_indexof($str1$$Register, $str2$$Register,
15488 $cnt1$$Register, $cnt2$$Register,
15489 (-1), $result$$Register,
15490 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15491 %}
15492 ins_pipe( pipe_slow );
15493 %}
15494
15495 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15496 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15497 %{
15498 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15499 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15500 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15501 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15502 ins_encode %{
15503 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15504 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15505 %}
15506 ins_pipe( pipe_slow );
15507 %}
15508
15509 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15510 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15511 %{
15512 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15513 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15514 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15515 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15516 ins_encode %{
15517 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15518 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15519 %}
15520 ins_pipe( pipe_slow );
15521 %}
15522
15523 // fast string equals
15524 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15525 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
15526 %{
15527 predicate(!VM_Version::supports_avx512vlbw());
15528 match(Set result (StrEquals (Binary str1 str2) cnt));
15529 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15530
15531 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15532 ins_encode %{
15533 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15534 $cnt$$Register, $result$$Register, $tmp3$$Register,
15535 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15536 %}
15537 ins_pipe( pipe_slow );
15538 %}
15539
15540 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15541 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
15542 %{
15543 predicate(VM_Version::supports_avx512vlbw());
15544 match(Set result (StrEquals (Binary str1 str2) cnt));
15545 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15546
15547 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15548 ins_encode %{
15549 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15550 $cnt$$Register, $result$$Register, $tmp3$$Register,
15551 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15552 %}
15553 ins_pipe( pipe_slow );
15554 %}
15555
15556 // fast array equals
15557 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15558 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15559 %{
15560 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15561 match(Set result (AryEq ary1 ary2));
15562 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15563
15564 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15565 ins_encode %{
15566 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15567 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15568 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15569 %}
15570 ins_pipe( pipe_slow );
15571 %}
15572
15573 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15574 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15575 %{
15576 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15577 match(Set result (AryEq ary1 ary2));
15578 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15579
15580 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15581 ins_encode %{
15582 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15583 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15584 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15585 %}
15586 ins_pipe( pipe_slow );
15587 %}
15588
15589 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15590 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15591 %{
15592 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15593 match(Set result (AryEq ary1 ary2));
15594 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15595
15596 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15597 ins_encode %{
15598 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15599 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15600 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
15601 %}
15602 ins_pipe( pipe_slow );
15603 %}
15604
15605 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15606 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15607 %{
15608 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15609 match(Set result (AryEq ary1 ary2));
15610 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15611
15612 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15613 ins_encode %{
15614 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15615 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15616 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
15617 %}
15618 ins_pipe( pipe_slow );
15619 %}
15620
15621 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
15622 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
15623 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
15624 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
15625 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
15626 %{
15627 predicate(UseAVX >= 2);
15628 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
15629 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
15630 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
15631 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
15632 USE basic_type, KILL cr);
15633
15634 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
15635 ins_encode %{
15636 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
15637 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15638 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
15639 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
15640 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
15641 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
15642 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
15643 %}
15644 ins_pipe( pipe_slow );
15645 %}
15646
15647 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15648 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
15649 %{
15650 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15651 match(Set result (CountPositives ary1 len));
15652 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15653
15654 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15655 ins_encode %{
15656 __ count_positives($ary1$$Register, $len$$Register,
15657 $result$$Register, $tmp3$$Register,
15658 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
15659 %}
15660 ins_pipe( pipe_slow );
15661 %}
15662
15663 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15664 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
15665 %{
15666 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15667 match(Set result (CountPositives ary1 len));
15668 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15669
15670 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15671 ins_encode %{
15672 __ count_positives($ary1$$Register, $len$$Register,
15673 $result$$Register, $tmp3$$Register,
15674 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
15675 %}
15676 ins_pipe( pipe_slow );
15677 %}
15678
15679 // fast char[] to byte[] compression
15680 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15681 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15682 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15683 match(Set result (StrCompressedCopy src (Binary dst len)));
15684 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
15685 USE_KILL len, KILL tmp5, KILL cr);
15686
15687 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15688 ins_encode %{
15689 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15690 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15691 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15692 knoreg, knoreg);
15693 %}
15694 ins_pipe( pipe_slow );
15695 %}
15696
15697 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15698 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15699 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15700 match(Set result (StrCompressedCopy src (Binary dst len)));
15701 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
15702 USE_KILL len, KILL tmp5, KILL cr);
15703
15704 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15705 ins_encode %{
15706 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15707 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15708 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15709 $ktmp1$$KRegister, $ktmp2$$KRegister);
15710 %}
15711 ins_pipe( pipe_slow );
15712 %}
15713 // fast byte[] to char[] inflation
15714 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15715 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
15716 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15717 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15718 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15719
15720 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15721 ins_encode %{
15722 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15723 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
15724 %}
15725 ins_pipe( pipe_slow );
15726 %}
15727
15728 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15729 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
15730 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15731 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15732 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15733
15734 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15735 ins_encode %{
15736 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15737 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
15738 %}
15739 ins_pipe( pipe_slow );
15740 %}
15741
15742 // encode char[] to byte[] in ISO_8859_1
15743 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15744 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15745 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15746 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
15747 match(Set result (EncodeISOArray src (Binary dst len)));
15748 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15749
15750 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15751 ins_encode %{
15752 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15753 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15754 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
15755 %}
15756 ins_pipe( pipe_slow );
15757 %}
15758
15759 // encode char[] to byte[] in ASCII
15760 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15761 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15762 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15763 predicate(((EncodeISOArrayNode*)n)->is_ascii());
15764 match(Set result (EncodeISOArray src (Binary dst len)));
15765 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15766
15767 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15768 ins_encode %{
15769 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15770 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15771 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
15772 %}
15773 ins_pipe( pipe_slow );
15774 %}
15775
15776 //----------Overflow Math Instructions-----------------------------------------
15777
15778 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15779 %{
15780 match(Set cr (OverflowAddI op1 op2));
15781 effect(DEF cr, USE_KILL op1, USE op2);
15782
15783 format %{ "addl $op1, $op2\t# overflow check int" %}
15784
15785 ins_encode %{
15786 __ addl($op1$$Register, $op2$$Register);
15787 %}
15788 ins_pipe(ialu_reg_reg);
15789 %}
15790
15791 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
15792 %{
15793 match(Set cr (OverflowAddI op1 op2));
15794 effect(DEF cr, USE_KILL op1, USE op2);
15795
15796 format %{ "addl $op1, $op2\t# overflow check int" %}
15797
15798 ins_encode %{
15799 __ addl($op1$$Register, $op2$$constant);
15800 %}
15801 ins_pipe(ialu_reg_reg);
15802 %}
15803
15804 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15805 %{
15806 match(Set cr (OverflowAddL op1 op2));
15807 effect(DEF cr, USE_KILL op1, USE op2);
15808
15809 format %{ "addq $op1, $op2\t# overflow check long" %}
15810 ins_encode %{
15811 __ addq($op1$$Register, $op2$$Register);
15812 %}
15813 ins_pipe(ialu_reg_reg);
15814 %}
15815
15816 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
15817 %{
15818 match(Set cr (OverflowAddL op1 op2));
15819 effect(DEF cr, USE_KILL op1, USE op2);
15820
15821 format %{ "addq $op1, $op2\t# overflow check long" %}
15822 ins_encode %{
15823 __ addq($op1$$Register, $op2$$constant);
15824 %}
15825 ins_pipe(ialu_reg_reg);
15826 %}
15827
15828 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15829 %{
15830 match(Set cr (OverflowSubI op1 op2));
15831
15832 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15833 ins_encode %{
15834 __ cmpl($op1$$Register, $op2$$Register);
15835 %}
15836 ins_pipe(ialu_reg_reg);
15837 %}
15838
15839 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15840 %{
15841 match(Set cr (OverflowSubI op1 op2));
15842
15843 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15844 ins_encode %{
15845 __ cmpl($op1$$Register, $op2$$constant);
15846 %}
15847 ins_pipe(ialu_reg_reg);
15848 %}
15849
15850 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15851 %{
15852 match(Set cr (OverflowSubL op1 op2));
15853
15854 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15855 ins_encode %{
15856 __ cmpq($op1$$Register, $op2$$Register);
15857 %}
15858 ins_pipe(ialu_reg_reg);
15859 %}
15860
15861 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15862 %{
15863 match(Set cr (OverflowSubL op1 op2));
15864
15865 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15866 ins_encode %{
15867 __ cmpq($op1$$Register, $op2$$constant);
15868 %}
15869 ins_pipe(ialu_reg_reg);
15870 %}
15871
15872 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
15873 %{
15874 match(Set cr (OverflowSubI zero op2));
15875 effect(DEF cr, USE_KILL op2);
15876
15877 format %{ "negl $op2\t# overflow check int" %}
15878 ins_encode %{
15879 __ negl($op2$$Register);
15880 %}
15881 ins_pipe(ialu_reg_reg);
15882 %}
15883
15884 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
15885 %{
15886 match(Set cr (OverflowSubL zero op2));
15887 effect(DEF cr, USE_KILL op2);
15888
15889 format %{ "negq $op2\t# overflow check long" %}
15890 ins_encode %{
15891 __ negq($op2$$Register);
15892 %}
15893 ins_pipe(ialu_reg_reg);
15894 %}
15895
15896 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15897 %{
15898 match(Set cr (OverflowMulI op1 op2));
15899 effect(DEF cr, USE_KILL op1, USE op2);
15900
15901 format %{ "imull $op1, $op2\t# overflow check int" %}
15902 ins_encode %{
15903 __ imull($op1$$Register, $op2$$Register);
15904 %}
15905 ins_pipe(ialu_reg_reg_alu0);
15906 %}
15907
15908 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
15909 %{
15910 match(Set cr (OverflowMulI op1 op2));
15911 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15912
15913 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
15914 ins_encode %{
15915 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
15916 %}
15917 ins_pipe(ialu_reg_reg_alu0);
15918 %}
15919
15920 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15921 %{
15922 match(Set cr (OverflowMulL op1 op2));
15923 effect(DEF cr, USE_KILL op1, USE op2);
15924
15925 format %{ "imulq $op1, $op2\t# overflow check long" %}
15926 ins_encode %{
15927 __ imulq($op1$$Register, $op2$$Register);
15928 %}
15929 ins_pipe(ialu_reg_reg_alu0);
15930 %}
15931
15932 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
15933 %{
15934 match(Set cr (OverflowMulL op1 op2));
15935 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15936
15937 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
15938 ins_encode %{
15939 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
15940 %}
15941 ins_pipe(ialu_reg_reg_alu0);
15942 %}
15943
15944
15945 //----------Control Flow Instructions------------------------------------------
15946 // Signed compare Instructions
15947
15948 // XXX more variants!!
15949 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15950 %{
15951 match(Set cr (CmpI op1 op2));
15952 effect(DEF cr, USE op1, USE op2);
15953
15954 format %{ "cmpl $op1, $op2" %}
15955 ins_encode %{
15956 __ cmpl($op1$$Register, $op2$$Register);
15957 %}
15958 ins_pipe(ialu_cr_reg_reg);
15959 %}
15960
15961 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15962 %{
15963 match(Set cr (CmpI op1 op2));
15964
15965 format %{ "cmpl $op1, $op2" %}
15966 ins_encode %{
15967 __ cmpl($op1$$Register, $op2$$constant);
15968 %}
15969 ins_pipe(ialu_cr_reg_imm);
15970 %}
15971
15972 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
15973 %{
15974 match(Set cr (CmpI op1 (LoadI op2)));
15975
15976 ins_cost(500); // XXX
15977 format %{ "cmpl $op1, $op2" %}
15978 ins_encode %{
15979 __ cmpl($op1$$Register, $op2$$Address);
15980 %}
15981 ins_pipe(ialu_cr_reg_mem);
15982 %}
15983
15984 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
15985 %{
15986 match(Set cr (CmpI src zero));
15987
15988 format %{ "testl $src, $src" %}
15989 ins_encode %{
15990 __ testl($src$$Register, $src$$Register);
15991 %}
15992 ins_pipe(ialu_cr_reg_imm);
15993 %}
15994
15995 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
15996 %{
15997 match(Set cr (CmpI (AndI src con) zero));
15998
15999 format %{ "testl $src, $con" %}
16000 ins_encode %{
16001 __ testl($src$$Register, $con$$constant);
16002 %}
16003 ins_pipe(ialu_cr_reg_imm);
16004 %}
16005
16006 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
16007 %{
16008 match(Set cr (CmpI (AndI src1 src2) zero));
16009
16010 format %{ "testl $src1, $src2" %}
16011 ins_encode %{
16012 __ testl($src1$$Register, $src2$$Register);
16013 %}
16014 ins_pipe(ialu_cr_reg_imm);
16015 %}
16016
16017 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
16018 %{
16019 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
16020
16021 format %{ "testl $src, $mem" %}
16022 ins_encode %{
16023 __ testl($src$$Register, $mem$$Address);
16024 %}
16025 ins_pipe(ialu_cr_reg_mem);
16026 %}
16027
16028 // Unsigned compare Instructions; really, same as signed except they
16029 // produce an rFlagsRegU instead of rFlagsReg.
16030 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
16031 %{
16032 match(Set cr (CmpU op1 op2));
16033
16034 format %{ "cmpl $op1, $op2\t# unsigned" %}
16035 ins_encode %{
16036 __ cmpl($op1$$Register, $op2$$Register);
16037 %}
16038 ins_pipe(ialu_cr_reg_reg);
16039 %}
16040
16041 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
16042 %{
16043 match(Set cr (CmpU op1 op2));
16044
16045 format %{ "cmpl $op1, $op2\t# unsigned" %}
16046 ins_encode %{
16047 __ cmpl($op1$$Register, $op2$$constant);
16048 %}
16049 ins_pipe(ialu_cr_reg_imm);
16050 %}
16051
16052 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
16053 %{
16054 match(Set cr (CmpU op1 (LoadI op2)));
16055
16056 ins_cost(500); // XXX
16057 format %{ "cmpl $op1, $op2\t# unsigned" %}
16058 ins_encode %{
16059 __ cmpl($op1$$Register, $op2$$Address);
16060 %}
16061 ins_pipe(ialu_cr_reg_mem);
16062 %}
16063
16064 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
16065 %{
16066 match(Set cr (CmpU src zero));
16067
16068 format %{ "testl $src, $src\t# unsigned" %}
16069 ins_encode %{
16070 __ testl($src$$Register, $src$$Register);
16071 %}
16072 ins_pipe(ialu_cr_reg_imm);
16073 %}
16074
16075 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
16076 %{
16077 match(Set cr (CmpP op1 op2));
16078
16079 format %{ "cmpq $op1, $op2\t# ptr" %}
16080 ins_encode %{
16081 __ cmpq($op1$$Register, $op2$$Register);
16082 %}
16083 ins_pipe(ialu_cr_reg_reg);
16084 %}
16085
16086 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
16087 %{
16088 match(Set cr (CmpP op1 (LoadP op2)));
16089 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16090
16091 ins_cost(500); // XXX
16092 format %{ "cmpq $op1, $op2\t# ptr" %}
16093 ins_encode %{
16094 __ cmpq($op1$$Register, $op2$$Address);
16095 %}
16096 ins_pipe(ialu_cr_reg_mem);
16097 %}
16098
16099 // XXX this is generalized by compP_rReg_mem???
16100 // Compare raw pointer (used in out-of-heap check).
16101 // Only works because non-oop pointers must be raw pointers
16102 // and raw pointers have no anti-dependencies.
16103 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
16104 %{
16105 predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
16106 n->in(2)->as_Load()->barrier_data() == 0);
16107 match(Set cr (CmpP op1 (LoadP op2)));
16108
16109 format %{ "cmpq $op1, $op2\t# raw ptr" %}
16110 ins_encode %{
16111 __ cmpq($op1$$Register, $op2$$Address);
16112 %}
16113 ins_pipe(ialu_cr_reg_mem);
16114 %}
16115
16116 // This will generate a signed flags result. This should be OK since
16117 // any compare to a zero should be eq/neq.
16118 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
16119 %{
16120 match(Set cr (CmpP src zero));
16121
16122 format %{ "testq $src, $src\t# ptr" %}
16123 ins_encode %{
16124 __ testq($src$$Register, $src$$Register);
16125 %}
16126 ins_pipe(ialu_cr_reg_imm);
16127 %}
16128
16129 // This will generate a signed flags result. This should be OK since
16130 // any compare to a zero should be eq/neq.
16131 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
16132 %{
16133 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
16134 n->in(1)->as_Load()->barrier_data() == 0);
16135 match(Set cr (CmpP (LoadP op) zero));
16136
16137 ins_cost(500); // XXX
16138 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
16139 ins_encode %{
16140 __ testq($op$$Address, 0xFFFFFFFF);
16141 %}
16142 ins_pipe(ialu_cr_reg_imm);
16143 %}
16144
16145 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
16146 %{
16147 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
16148 n->in(1)->as_Load()->barrier_data() == 0);
16149 match(Set cr (CmpP (LoadP mem) zero));
16150
16151 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
16152 ins_encode %{
16153 __ cmpq(r12, $mem$$Address);
16154 %}
16155 ins_pipe(ialu_cr_reg_mem);
16156 %}
16157
16158 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
16159 %{
16160 match(Set cr (CmpN op1 op2));
16161
16162 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
16163 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
16164 ins_pipe(ialu_cr_reg_reg);
16165 %}
16166
16167 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
16168 %{
16169 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16170 match(Set cr (CmpN src (LoadN mem)));
16171
16172 format %{ "cmpl $src, $mem\t# compressed ptr" %}
16173 ins_encode %{
16174 __ cmpl($src$$Register, $mem$$Address);
16175 %}
16176 ins_pipe(ialu_cr_reg_mem);
16177 %}
16178
16179 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
16180 match(Set cr (CmpN op1 op2));
16181
16182 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
16183 ins_encode %{
16184 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
16185 %}
16186 ins_pipe(ialu_cr_reg_imm);
16187 %}
16188
16189 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
16190 %{
16191 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16192 match(Set cr (CmpN src (LoadN mem)));
16193
16194 format %{ "cmpl $mem, $src\t# compressed ptr" %}
16195 ins_encode %{
16196 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
16197 %}
16198 ins_pipe(ialu_cr_reg_mem);
16199 %}
16200
16201 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
16202 match(Set cr (CmpN op1 op2));
16203
16204 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
16205 ins_encode %{
16206 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
16207 %}
16208 ins_pipe(ialu_cr_reg_imm);
16209 %}
16210
16211 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
16212 %{
16213 predicate(!UseCompactObjectHeaders);
16214 match(Set cr (CmpN src (LoadNKlass mem)));
16215
16216 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
16217 ins_encode %{
16218 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
16219 %}
16220 ins_pipe(ialu_cr_reg_mem);
16221 %}
16222
16223 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
16224 match(Set cr (CmpN src zero));
16225
16226 format %{ "testl $src, $src\t# compressed ptr" %}
16227 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
16228 ins_pipe(ialu_cr_reg_imm);
16229 %}
16230
16231 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
16232 %{
16233 predicate(CompressedOops::base() != nullptr &&
16234 n->in(1)->as_Load()->barrier_data() == 0);
16235 match(Set cr (CmpN (LoadN mem) zero));
16236
16237 ins_cost(500); // XXX
16238 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
16239 ins_encode %{
16240 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
16241 %}
16242 ins_pipe(ialu_cr_reg_mem);
16243 %}
16244
16245 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
16246 %{
16247 predicate(CompressedOops::base() == nullptr &&
16248 n->in(1)->as_Load()->barrier_data() == 0);
16249 match(Set cr (CmpN (LoadN mem) zero));
16250
16251 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
16252 ins_encode %{
16253 __ cmpl(r12, $mem$$Address);
16254 %}
16255 ins_pipe(ialu_cr_reg_mem);
16256 %}
16257
16258 // Yanked all unsigned pointer compare operations.
16259 // Pointer compares are done with CmpP which is already unsigned.
16260
16261 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
16262 %{
16263 match(Set cr (CmpL op1 op2));
16264
16265 format %{ "cmpq $op1, $op2" %}
16266 ins_encode %{
16267 __ cmpq($op1$$Register, $op2$$Register);
16268 %}
16269 ins_pipe(ialu_cr_reg_reg);
16270 %}
16271
16272 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
16273 %{
16274 match(Set cr (CmpL op1 op2));
16275
16276 format %{ "cmpq $op1, $op2" %}
16277 ins_encode %{
16278 __ cmpq($op1$$Register, $op2$$constant);
16279 %}
16280 ins_pipe(ialu_cr_reg_imm);
16281 %}
16282
16283 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
16284 %{
16285 match(Set cr (CmpL op1 (LoadL op2)));
16286
16287 format %{ "cmpq $op1, $op2" %}
16288 ins_encode %{
16289 __ cmpq($op1$$Register, $op2$$Address);
16290 %}
16291 ins_pipe(ialu_cr_reg_mem);
16292 %}
16293
16294 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
16295 %{
16296 match(Set cr (CmpL src zero));
16297
16298 format %{ "testq $src, $src" %}
16299 ins_encode %{
16300 __ testq($src$$Register, $src$$Register);
16301 %}
16302 ins_pipe(ialu_cr_reg_imm);
16303 %}
16304
16305 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
16306 %{
16307 match(Set cr (CmpL (AndL src con) zero));
16308
16309 format %{ "testq $src, $con\t# long" %}
16310 ins_encode %{
16311 __ testq($src$$Register, $con$$constant);
16312 %}
16313 ins_pipe(ialu_cr_reg_imm);
16314 %}
16315
16316 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
16317 %{
16318 match(Set cr (CmpL (AndL src1 src2) zero));
16319
16320 format %{ "testq $src1, $src2\t# long" %}
16321 ins_encode %{
16322 __ testq($src1$$Register, $src2$$Register);
16323 %}
16324 ins_pipe(ialu_cr_reg_imm);
16325 %}
16326
16327 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
16328 %{
16329 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
16330
16331 format %{ "testq $src, $mem" %}
16332 ins_encode %{
16333 __ testq($src$$Register, $mem$$Address);
16334 %}
16335 ins_pipe(ialu_cr_reg_mem);
16336 %}
16337
16338 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
16339 %{
16340 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
16341
16342 format %{ "testq $src, $mem" %}
16343 ins_encode %{
16344 __ testq($src$$Register, $mem$$Address);
16345 %}
16346 ins_pipe(ialu_cr_reg_mem);
16347 %}
16348
16349 // Manifest a CmpU result in an integer register. Very painful.
16350 // This is the test to avoid.
16351 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
16352 %{
16353 match(Set dst (CmpU3 src1 src2));
16354 effect(KILL flags);
16355
16356 ins_cost(275); // XXX
16357 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
16358 "movl $dst, -1\n\t"
16359 "jb,u done\n\t"
16360 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16361 "done:" %}
16362 ins_encode %{
16363 Label done;
16364 __ cmpl($src1$$Register, $src2$$Register);
16365 __ movl($dst$$Register, -1);
16366 __ jccb(Assembler::below, done);
16367 __ setcc(Assembler::notZero, $dst$$Register);
16368 __ bind(done);
16369 %}
16370 ins_pipe(pipe_slow);
16371 %}
16372
16373 // Manifest a CmpL result in an integer register. Very painful.
16374 // This is the test to avoid.
16375 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16376 %{
16377 match(Set dst (CmpL3 src1 src2));
16378 effect(KILL flags);
16379
16380 ins_cost(275); // XXX
16381 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16382 "movl $dst, -1\n\t"
16383 "jl,s done\n\t"
16384 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16385 "done:" %}
16386 ins_encode %{
16387 Label done;
16388 __ cmpq($src1$$Register, $src2$$Register);
16389 __ movl($dst$$Register, -1);
16390 __ jccb(Assembler::less, done);
16391 __ setcc(Assembler::notZero, $dst$$Register);
16392 __ bind(done);
16393 %}
16394 ins_pipe(pipe_slow);
16395 %}
16396
16397 // Manifest a CmpUL result in an integer register. Very painful.
16398 // This is the test to avoid.
16399 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16400 %{
16401 match(Set dst (CmpUL3 src1 src2));
16402 effect(KILL flags);
16403
16404 ins_cost(275); // XXX
16405 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16406 "movl $dst, -1\n\t"
16407 "jb,u done\n\t"
16408 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16409 "done:" %}
16410 ins_encode %{
16411 Label done;
16412 __ cmpq($src1$$Register, $src2$$Register);
16413 __ movl($dst$$Register, -1);
16414 __ jccb(Assembler::below, done);
16415 __ setcc(Assembler::notZero, $dst$$Register);
16416 __ bind(done);
16417 %}
16418 ins_pipe(pipe_slow);
16419 %}
16420
16421 // Unsigned long compare Instructions; really, same as signed long except they
16422 // produce an rFlagsRegU instead of rFlagsReg.
16423 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
16424 %{
16425 match(Set cr (CmpUL op1 op2));
16426
16427 format %{ "cmpq $op1, $op2\t# unsigned" %}
16428 ins_encode %{
16429 __ cmpq($op1$$Register, $op2$$Register);
16430 %}
16431 ins_pipe(ialu_cr_reg_reg);
16432 %}
16433
16434 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
16435 %{
16436 match(Set cr (CmpUL op1 op2));
16437
16438 format %{ "cmpq $op1, $op2\t# unsigned" %}
16439 ins_encode %{
16440 __ cmpq($op1$$Register, $op2$$constant);
16441 %}
16442 ins_pipe(ialu_cr_reg_imm);
16443 %}
16444
16445 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
16446 %{
16447 match(Set cr (CmpUL op1 (LoadL op2)));
16448
16449 format %{ "cmpq $op1, $op2\t# unsigned" %}
16450 ins_encode %{
16451 __ cmpq($op1$$Register, $op2$$Address);
16452 %}
16453 ins_pipe(ialu_cr_reg_mem);
16454 %}
16455
16456 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
16457 %{
16458 match(Set cr (CmpUL src zero));
16459
16460 format %{ "testq $src, $src\t# unsigned" %}
16461 ins_encode %{
16462 __ testq($src$$Register, $src$$Register);
16463 %}
16464 ins_pipe(ialu_cr_reg_imm);
16465 %}
16466
16467 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
16468 %{
16469 match(Set cr (CmpI (LoadB mem) imm));
16470
16471 ins_cost(125);
16472 format %{ "cmpb $mem, $imm" %}
16473 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
16474 ins_pipe(ialu_cr_reg_mem);
16475 %}
16476
16477 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
16478 %{
16479 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
16480
16481 ins_cost(125);
16482 format %{ "testb $mem, $imm\t# ubyte" %}
16483 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16484 ins_pipe(ialu_cr_reg_mem);
16485 %}
16486
16487 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
16488 %{
16489 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
16490
16491 ins_cost(125);
16492 format %{ "testb $mem, $imm\t# byte" %}
16493 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16494 ins_pipe(ialu_cr_reg_mem);
16495 %}
16496
16497 //----------Max and Min--------------------------------------------------------
16498 // Min Instructions
16499
16500 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
16501 %{
16502 predicate(!UseAPX);
16503 effect(USE_DEF dst, USE src, USE cr);
16504
16505 format %{ "cmovlgt $dst, $src\t# min" %}
16506 ins_encode %{
16507 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
16508 %}
16509 ins_pipe(pipe_cmov_reg);
16510 %}
16511
16512 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16513 %{
16514 predicate(UseAPX);
16515 effect(DEF dst, USE src1, USE src2, USE cr);
16516
16517 format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
16518 ins_encode %{
16519 __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
16520 %}
16521 ins_pipe(pipe_cmov_reg);
16522 %}
16523
16524 instruct minI_rReg(rRegI dst, rRegI src)
16525 %{
16526 predicate(!UseAPX);
16527 match(Set dst (MinI dst src));
16528
16529 ins_cost(200);
16530 expand %{
16531 rFlagsReg cr;
16532 compI_rReg(cr, dst, src);
16533 cmovI_reg_g(dst, src, cr);
16534 %}
16535 %}
16536
16537 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16538 %{
16539 predicate(UseAPX);
16540 match(Set dst (MinI src1 src2));
16541 effect(DEF dst, USE src1, USE src2);
16542 flag(PD::Flag_ndd_demotable_opr1);
16543
16544 ins_cost(200);
16545 expand %{
16546 rFlagsReg cr;
16547 compI_rReg(cr, src1, src2);
16548 cmovI_reg_g_ndd(dst, src1, src2, cr);
16549 %}
16550 %}
16551
16552 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
16553 %{
16554 predicate(!UseAPX);
16555 effect(USE_DEF dst, USE src, USE cr);
16556
16557 format %{ "cmovllt $dst, $src\t# max" %}
16558 ins_encode %{
16559 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
16560 %}
16561 ins_pipe(pipe_cmov_reg);
16562 %}
16563
16564 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16565 %{
16566 predicate(UseAPX);
16567 effect(DEF dst, USE src1, USE src2, USE cr);
16568
16569 format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
16570 ins_encode %{
16571 __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
16572 %}
16573 ins_pipe(pipe_cmov_reg);
16574 %}
16575
16576 instruct maxI_rReg(rRegI dst, rRegI src)
16577 %{
16578 predicate(!UseAPX);
16579 match(Set dst (MaxI dst src));
16580
16581 ins_cost(200);
16582 expand %{
16583 rFlagsReg cr;
16584 compI_rReg(cr, dst, src);
16585 cmovI_reg_l(dst, src, cr);
16586 %}
16587 %}
16588
16589 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16590 %{
16591 predicate(UseAPX);
16592 match(Set dst (MaxI src1 src2));
16593 effect(DEF dst, USE src1, USE src2);
16594 flag(PD::Flag_ndd_demotable_opr1);
16595
16596 ins_cost(200);
16597 expand %{
16598 rFlagsReg cr;
16599 compI_rReg(cr, src1, src2);
16600 cmovI_reg_l_ndd(dst, src1, src2, cr);
16601 %}
16602 %}
16603
16604 // ============================================================================
16605 // Branch Instructions
16606
16607 // Jump Direct - Label defines a relative address from JMP+1
16608 instruct jmpDir(label labl)
16609 %{
16610 match(Goto);
16611 effect(USE labl);
16612
16613 ins_cost(300);
16614 format %{ "jmp $labl" %}
16615 size(5);
16616 ins_encode %{
16617 Label* L = $labl$$label;
16618 __ jmp(*L, false); // Always long jump
16619 %}
16620 ins_pipe(pipe_jmp);
16621 %}
16622
16623 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16624 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
16625 %{
16626 match(If cop cr);
16627 effect(USE labl);
16628
16629 ins_cost(300);
16630 format %{ "j$cop $labl" %}
16631 size(6);
16632 ins_encode %{
16633 Label* L = $labl$$label;
16634 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16635 %}
16636 ins_pipe(pipe_jcc);
16637 %}
16638
16639 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16640 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
16641 %{
16642 match(CountedLoopEnd cop cr);
16643 effect(USE labl);
16644
16645 ins_cost(300);
16646 format %{ "j$cop $labl\t# loop end" %}
16647 size(6);
16648 ins_encode %{
16649 Label* L = $labl$$label;
16650 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16651 %}
16652 ins_pipe(pipe_jcc);
16653 %}
16654
16655 // Jump Direct Conditional - using unsigned comparison
16656 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16657 match(If cop cmp);
16658 effect(USE labl);
16659
16660 ins_cost(300);
16661 format %{ "j$cop,u $labl" %}
16662 size(6);
16663 ins_encode %{
16664 Label* L = $labl$$label;
16665 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16666 %}
16667 ins_pipe(pipe_jcc);
16668 %}
16669
16670 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16671 match(If cop cmp);
16672 effect(USE labl);
16673
16674 ins_cost(200);
16675 format %{ "j$cop,u $labl" %}
16676 size(6);
16677 ins_encode %{
16678 Label* L = $labl$$label;
16679 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16680 %}
16681 ins_pipe(pipe_jcc);
16682 %}
16683
16684 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16685 match(If cop cmp);
16686 effect(USE labl);
16687
16688 ins_cost(200);
16689 format %{ $$template
16690 if ($cop$$cmpcode == Assembler::notEqual) {
16691 $$emit$$"jp,u $labl\n\t"
16692 $$emit$$"j$cop,u $labl"
16693 } else {
16694 $$emit$$"jp,u done\n\t"
16695 $$emit$$"j$cop,u $labl\n\t"
16696 $$emit$$"done:"
16697 }
16698 %}
16699 ins_encode %{
16700 Label* l = $labl$$label;
16701 if ($cop$$cmpcode == Assembler::notEqual) {
16702 __ jcc(Assembler::parity, *l, false);
16703 __ jcc(Assembler::notEqual, *l, false);
16704 } else if ($cop$$cmpcode == Assembler::equal) {
16705 Label done;
16706 __ jccb(Assembler::parity, done);
16707 __ jcc(Assembler::equal, *l, false);
16708 __ bind(done);
16709 } else {
16710 ShouldNotReachHere();
16711 }
16712 %}
16713 ins_pipe(pipe_jcc);
16714 %}
16715
16716 // Jump Direct Conditional - using signed and unsigned comparison
16717 instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16718 match(If cop cmp);
16719 effect(USE labl);
16720
16721 ins_cost(200);
16722 format %{ "j$cop,su $labl" %}
16723 size(6);
16724 ins_encode %{
16725 Label* L = $labl$$label;
16726 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16727 %}
16728 ins_pipe(pipe_jcc);
16729 %}
16730
16731 // ============================================================================
16732 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
16733 // superklass array for an instance of the superklass. Set a hidden
16734 // internal cache on a hit (cache is checked with exposed code in
16735 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16736 // encoding ALSO sets flags.
16737
16738 instruct partialSubtypeCheck(rdi_RegP result,
16739 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
16740 rFlagsReg cr)
16741 %{
16742 match(Set result (PartialSubtypeCheck sub super));
16743 predicate(!UseSecondarySupersTable);
16744 effect(KILL rcx, KILL cr);
16745
16746 ins_cost(1100); // slightly larger than the next version
16747 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
16748 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
16749 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
16750 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
16751 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
16752 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
16753 "xorq $result, $result\t\t Hit: rdi zero\n\t"
16754 "miss:\t" %}
16755
16756 ins_encode %{
16757 Label miss;
16758 // NB: Callers may assume that, when $result is a valid register,
16759 // check_klass_subtype_slow_path_linear sets it to a nonzero
16760 // value.
16761 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
16762 $rcx$$Register, $result$$Register,
16763 nullptr, &miss,
16764 /*set_cond_codes:*/ true);
16765 __ xorptr($result$$Register, $result$$Register);
16766 __ bind(miss);
16767 %}
16768
16769 ins_pipe(pipe_slow);
16770 %}
16771
16772 // ============================================================================
16773 // Two versions of hashtable-based partialSubtypeCheck, both used when
16774 // we need to search for a super class in the secondary supers array.
16775 // The first is used when we don't know _a priori_ the class being
16776 // searched for. The second, far more common, is used when we do know:
16777 // this is used for instanceof, checkcast, and any case where C2 can
16778 // determine it by constant propagation.
16779
16780 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
16781 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16782 rFlagsReg cr)
16783 %{
16784 match(Set result (PartialSubtypeCheck sub super));
16785 predicate(UseSecondarySupersTable);
16786 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16787
16788 ins_cost(1000);
16789 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16790
16791 ins_encode %{
16792 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
16793 $temp3$$Register, $temp4$$Register, $result$$Register);
16794 %}
16795
16796 ins_pipe(pipe_slow);
16797 %}
16798
16799 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
16800 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16801 rFlagsReg cr)
16802 %{
16803 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16804 predicate(UseSecondarySupersTable);
16805 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16806
16807 ins_cost(700); // smaller than the next version
16808 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16809
16810 ins_encode %{
16811 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16812 if (InlineSecondarySupersTest) {
16813 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
16814 $temp3$$Register, $temp4$$Register, $result$$Register,
16815 super_klass_slot);
16816 } else {
16817 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16818 }
16819 %}
16820
16821 ins_pipe(pipe_slow);
16822 %}
16823
16824 // ============================================================================
16825 // Branch Instructions -- short offset versions
16826 //
16827 // These instructions are used to replace jumps of a long offset (the default
16828 // match) with jumps of a shorter offset. These instructions are all tagged
16829 // with the ins_short_branch attribute, which causes the ADLC to suppress the
16830 // match rules in general matching. Instead, the ADLC generates a conversion
16831 // method in the MachNode which can be used to do in-place replacement of the
16832 // long variant with the shorter variant. The compiler will determine if a
16833 // branch can be taken by the is_short_branch_offset() predicate in the machine
16834 // specific code section of the file.
16835
16836 // Jump Direct - Label defines a relative address from JMP+1
16837 instruct jmpDir_short(label labl) %{
16838 match(Goto);
16839 effect(USE labl);
16840
16841 ins_cost(300);
16842 format %{ "jmp,s $labl" %}
16843 size(2);
16844 ins_encode %{
16845 Label* L = $labl$$label;
16846 __ jmpb(*L);
16847 %}
16848 ins_pipe(pipe_jmp);
16849 ins_short_branch(1);
16850 %}
16851
16852 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16853 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
16854 match(If cop cr);
16855 effect(USE labl);
16856
16857 ins_cost(300);
16858 format %{ "j$cop,s $labl" %}
16859 size(2);
16860 ins_encode %{
16861 Label* L = $labl$$label;
16862 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16863 %}
16864 ins_pipe(pipe_jcc);
16865 ins_short_branch(1);
16866 %}
16867
16868 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16869 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
16870 match(CountedLoopEnd cop cr);
16871 effect(USE labl);
16872
16873 ins_cost(300);
16874 format %{ "j$cop,s $labl\t# loop end" %}
16875 size(2);
16876 ins_encode %{
16877 Label* L = $labl$$label;
16878 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16879 %}
16880 ins_pipe(pipe_jcc);
16881 ins_short_branch(1);
16882 %}
16883
16884 // Jump Direct Conditional - using unsigned comparison
16885 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16886 match(If cop cmp);
16887 effect(USE labl);
16888
16889 ins_cost(300);
16890 format %{ "j$cop,us $labl" %}
16891 size(2);
16892 ins_encode %{
16893 Label* L = $labl$$label;
16894 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16895 %}
16896 ins_pipe(pipe_jcc);
16897 ins_short_branch(1);
16898 %}
16899
16900 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16901 match(If cop cmp);
16902 effect(USE labl);
16903
16904 ins_cost(300);
16905 format %{ "j$cop,us $labl" %}
16906 size(2);
16907 ins_encode %{
16908 Label* L = $labl$$label;
16909 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16910 %}
16911 ins_pipe(pipe_jcc);
16912 ins_short_branch(1);
16913 %}
16914
16915 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16916 match(If cop cmp);
16917 effect(USE labl);
16918
16919 ins_cost(300);
16920 format %{ $$template
16921 if ($cop$$cmpcode == Assembler::notEqual) {
16922 $$emit$$"jp,u,s $labl\n\t"
16923 $$emit$$"j$cop,u,s $labl"
16924 } else {
16925 $$emit$$"jp,u,s done\n\t"
16926 $$emit$$"j$cop,u,s $labl\n\t"
16927 $$emit$$"done:"
16928 }
16929 %}
16930 size(4);
16931 ins_encode %{
16932 Label* l = $labl$$label;
16933 if ($cop$$cmpcode == Assembler::notEqual) {
16934 __ jccb(Assembler::parity, *l);
16935 __ jccb(Assembler::notEqual, *l);
16936 } else if ($cop$$cmpcode == Assembler::equal) {
16937 Label done;
16938 __ jccb(Assembler::parity, done);
16939 __ jccb(Assembler::equal, *l);
16940 __ bind(done);
16941 } else {
16942 ShouldNotReachHere();
16943 }
16944 %}
16945 ins_pipe(pipe_jcc);
16946 ins_short_branch(1);
16947 %}
16948
16949 // Jump Direct Conditional - using signed and unsigned comparison
16950 instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16951 match(If cop cmp);
16952 effect(USE labl);
16953
16954 ins_cost(300);
16955 format %{ "j$cop,sus $labl" %}
16956 size(2);
16957 ins_encode %{
16958 Label* L = $labl$$label;
16959 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16960 %}
16961 ins_pipe(pipe_jcc);
16962 ins_short_branch(1);
16963 %}
16964
16965 // ============================================================================
16966 // inlined locking and unlocking
16967
16968 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
16969 match(Set cr (FastLock object box));
16970 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
16971 ins_cost(300);
16972 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
16973 ins_encode %{
16974 __ fast_lock($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16975 %}
16976 ins_pipe(pipe_slow);
16977 %}
16978
16979 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
16980 match(Set cr (FastUnlock object rax_reg));
16981 effect(TEMP tmp, USE_KILL rax_reg);
16982 ins_cost(300);
16983 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
16984 ins_encode %{
16985 __ fast_unlock($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16986 %}
16987 ins_pipe(pipe_slow);
16988 %}
16989
16990
16991 // ============================================================================
16992 // Safepoint Instructions
16993 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
16994 %{
16995 match(SafePoint poll);
16996 effect(KILL cr, USE poll);
16997
16998 format %{ "testl rax, [$poll]\t"
16999 "# Safepoint: poll for GC" %}
17000 ins_cost(125);
17001 ins_encode %{
17002 __ relocate(relocInfo::poll_type);
17003 address pre_pc = __ pc();
17004 __ testl(rax, Address($poll$$Register, 0));
17005 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
17006 %}
17007 ins_pipe(ialu_reg_mem);
17008 %}
17009
17010 instruct mask_all_evexL(kReg dst, rRegL src) %{
17011 match(Set dst (MaskAll src));
17012 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
17013 ins_encode %{
17014 int mask_len = Matcher::vector_length(this);
17015 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
17016 %}
17017 ins_pipe( pipe_slow );
17018 %}
17019
17020 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
17021 predicate(Matcher::vector_length(n) > 32);
17022 match(Set dst (MaskAll src));
17023 effect(TEMP tmp);
17024 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
17025 ins_encode %{
17026 int mask_len = Matcher::vector_length(this);
17027 __ movslq($tmp$$Register, $src$$Register);
17028 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
17029 %}
17030 ins_pipe( pipe_slow );
17031 %}
17032
17033 // ============================================================================
17034 // Procedure Call/Return Instructions
17035 // Call Java Static Instruction
17036 // Note: If this code changes, the corresponding ret_addr_offset() and
17037 // compute_padding() functions will have to be adjusted.
17038 instruct CallStaticJavaDirect(method meth) %{
17039 match(CallStaticJava);
17040 effect(USE meth);
17041
17042 ins_cost(300);
17043 format %{ "call,static " %}
17044 opcode(0xE8); /* E8 cd */
17045 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
17046 ins_pipe(pipe_slow);
17047 ins_alignment(4);
17048 %}
17049
17050 // Call Java Dynamic Instruction
17051 // Note: If this code changes, the corresponding ret_addr_offset() and
17052 // compute_padding() functions will have to be adjusted.
17053 instruct CallDynamicJavaDirect(method meth)
17054 %{
17055 match(CallDynamicJava);
17056 effect(USE meth);
17057
17058 ins_cost(300);
17059 format %{ "movq rax, #Universe::non_oop_word()\n\t"
17060 "call,dynamic " %}
17061 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
17062 ins_pipe(pipe_slow);
17063 ins_alignment(4);
17064 %}
17065
17066 // Call Runtime Instruction
17067 instruct CallRuntimeDirect(method meth)
17068 %{
17069 match(CallRuntime);
17070 effect(USE meth);
17071
17072 ins_cost(300);
17073 format %{ "call,runtime " %}
17074 ins_encode(clear_avx, Java_To_Runtime(meth));
17075 ins_pipe(pipe_slow);
17076 %}
17077
17078 // Call runtime without safepoint
17079 instruct CallLeafDirect(method meth)
17080 %{
17081 match(CallLeaf);
17082 effect(USE meth);
17083
17084 ins_cost(300);
17085 format %{ "call_leaf,runtime " %}
17086 ins_encode(clear_avx, Java_To_Runtime(meth));
17087 ins_pipe(pipe_slow);
17088 %}
17089
17090 // Call runtime without safepoint and with vector arguments
17091 instruct CallLeafDirectVector(method meth)
17092 %{
17093 match(CallLeafVector);
17094 effect(USE meth);
17095
17096 ins_cost(300);
17097 format %{ "call_leaf,vector " %}
17098 ins_encode(Java_To_Runtime(meth));
17099 ins_pipe(pipe_slow);
17100 %}
17101
17102 // Call runtime without safepoint
17103 // entry point is null, target holds the address to call
17104 instruct CallLeafNoFPInDirect(rRegP target)
17105 %{
17106 predicate(n->as_Call()->entry_point() == nullptr);
17107 match(CallLeafNoFP target);
17108
17109 ins_cost(300);
17110 format %{ "call_leaf_nofp,runtime indirect " %}
17111 ins_encode %{
17112 __ call($target$$Register);
17113 %}
17114
17115 ins_pipe(pipe_slow);
17116 %}
17117
17118 // Call runtime without safepoint
17119 instruct CallLeafNoFPDirect(method meth)
17120 %{
17121 predicate(n->as_Call()->entry_point() != nullptr);
17122 match(CallLeafNoFP);
17123 effect(USE meth);
17124
17125 ins_cost(300);
17126 format %{ "call_leaf_nofp,runtime " %}
17127 ins_encode(clear_avx, Java_To_Runtime(meth));
17128 ins_pipe(pipe_slow);
17129 %}
17130
17131 // Return Instruction
17132 // Remove the return address & jump to it.
17133 // Notice: We always emit a nop after a ret to make sure there is room
17134 // for safepoint patching
17135 instruct Ret()
17136 %{
17137 match(Return);
17138
17139 format %{ "ret" %}
17140 ins_encode %{
17141 __ ret(0);
17142 %}
17143 ins_pipe(pipe_jmp);
17144 %}
17145
17146 // Tail Call; Jump from runtime stub to Java code.
17147 // Also known as an 'interprocedural jump'.
17148 // Target of jump will eventually return to caller.
17149 // TailJump below removes the return address.
17150 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
17151 // emitted just above the TailCall which has reset rbp to the caller state.
17152 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
17153 %{
17154 match(TailCall jump_target method_ptr);
17155
17156 ins_cost(300);
17157 format %{ "jmp $jump_target\t# rbx holds method" %}
17158 ins_encode %{
17159 __ jmp($jump_target$$Register);
17160 %}
17161 ins_pipe(pipe_jmp);
17162 %}
17163
17164 // Tail Jump; remove the return address; jump to target.
17165 // TailCall above leaves the return address around.
17166 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
17167 %{
17168 match(TailJump jump_target ex_oop);
17169
17170 ins_cost(300);
17171 format %{ "popq rdx\t# pop return address\n\t"
17172 "jmp $jump_target" %}
17173 ins_encode %{
17174 __ popq(as_Register(RDX_enc));
17175 __ jmp($jump_target$$Register);
17176 %}
17177 ins_pipe(pipe_jmp);
17178 %}
17179
17180 // Forward exception.
17181 instruct ForwardExceptionjmp()
17182 %{
17183 match(ForwardException);
17184
17185 format %{ "jmp forward_exception_stub" %}
17186 ins_encode %{
17187 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
17188 %}
17189 ins_pipe(pipe_jmp);
17190 %}
17191
17192 // Create exception oop: created by stack-crawling runtime code.
17193 // Created exception is now available to this handler, and is setup
17194 // just prior to jumping to this handler. No code emitted.
17195 instruct CreateException(rax_RegP ex_oop)
17196 %{
17197 match(Set ex_oop (CreateEx));
17198
17199 size(0);
17200 // use the following format syntax
17201 format %{ "# exception oop is in rax; no code emitted" %}
17202 ins_encode();
17203 ins_pipe(empty);
17204 %}
17205
17206 // Rethrow exception:
17207 // The exception oop will come in the first argument position.
17208 // Then JUMP (not call) to the rethrow stub code.
17209 instruct RethrowException()
17210 %{
17211 match(Rethrow);
17212
17213 // use the following format syntax
17214 format %{ "jmp rethrow_stub" %}
17215 ins_encode %{
17216 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
17217 %}
17218 ins_pipe(pipe_jmp);
17219 %}
17220
17221 // ============================================================================
17222 // This name is KNOWN by the ADLC and cannot be changed.
17223 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17224 // for this guy.
17225 instruct tlsLoadP(r15_RegP dst) %{
17226 match(Set dst (ThreadLocal));
17227 effect(DEF dst);
17228
17229 size(0);
17230 format %{ "# TLS is in R15" %}
17231 ins_encode( /*empty encoding*/ );
17232 ins_pipe(ialu_reg_reg);
17233 %}
17234
17235 instruct addF_reg(regF dst, regF src) %{
17236 predicate(UseAVX == 0);
17237 match(Set dst (AddF dst src));
17238
17239 format %{ "addss $dst, $src" %}
17240 ins_cost(150);
17241 ins_encode %{
17242 __ addss($dst$$XMMRegister, $src$$XMMRegister);
17243 %}
17244 ins_pipe(pipe_slow);
17245 %}
17246
17247 instruct addF_mem(regF dst, memory src) %{
17248 predicate(UseAVX == 0);
17249 match(Set dst (AddF dst (LoadF src)));
17250
17251 format %{ "addss $dst, $src" %}
17252 ins_cost(150);
17253 ins_encode %{
17254 __ addss($dst$$XMMRegister, $src$$Address);
17255 %}
17256 ins_pipe(pipe_slow);
17257 %}
17258
17259 instruct addF_imm(regF dst, immF con) %{
17260 predicate(UseAVX == 0);
17261 match(Set dst (AddF dst con));
17262 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17263 ins_cost(150);
17264 ins_encode %{
17265 __ addss($dst$$XMMRegister, $constantaddress($con));
17266 %}
17267 ins_pipe(pipe_slow);
17268 %}
17269
17270 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
17271 predicate(UseAVX > 0);
17272 match(Set dst (AddF src1 src2));
17273
17274 format %{ "vaddss $dst, $src1, $src2" %}
17275 ins_cost(150);
17276 ins_encode %{
17277 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17278 %}
17279 ins_pipe(pipe_slow);
17280 %}
17281
17282 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
17283 predicate(UseAVX > 0);
17284 match(Set dst (AddF src1 (LoadF src2)));
17285
17286 format %{ "vaddss $dst, $src1, $src2" %}
17287 ins_cost(150);
17288 ins_encode %{
17289 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17290 %}
17291 ins_pipe(pipe_slow);
17292 %}
17293
17294 instruct addF_reg_imm(regF dst, regF src, immF con) %{
17295 predicate(UseAVX > 0);
17296 match(Set dst (AddF src con));
17297
17298 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17299 ins_cost(150);
17300 ins_encode %{
17301 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17302 %}
17303 ins_pipe(pipe_slow);
17304 %}
17305
17306 instruct addD_reg(regD dst, regD src) %{
17307 predicate(UseAVX == 0);
17308 match(Set dst (AddD dst src));
17309
17310 format %{ "addsd $dst, $src" %}
17311 ins_cost(150);
17312 ins_encode %{
17313 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
17314 %}
17315 ins_pipe(pipe_slow);
17316 %}
17317
17318 instruct addD_mem(regD dst, memory src) %{
17319 predicate(UseAVX == 0);
17320 match(Set dst (AddD dst (LoadD src)));
17321
17322 format %{ "addsd $dst, $src" %}
17323 ins_cost(150);
17324 ins_encode %{
17325 __ addsd($dst$$XMMRegister, $src$$Address);
17326 %}
17327 ins_pipe(pipe_slow);
17328 %}
17329
17330 instruct addD_imm(regD dst, immD con) %{
17331 predicate(UseAVX == 0);
17332 match(Set dst (AddD dst con));
17333 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17334 ins_cost(150);
17335 ins_encode %{
17336 __ addsd($dst$$XMMRegister, $constantaddress($con));
17337 %}
17338 ins_pipe(pipe_slow);
17339 %}
17340
17341 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
17342 predicate(UseAVX > 0);
17343 match(Set dst (AddD src1 src2));
17344
17345 format %{ "vaddsd $dst, $src1, $src2" %}
17346 ins_cost(150);
17347 ins_encode %{
17348 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17349 %}
17350 ins_pipe(pipe_slow);
17351 %}
17352
17353 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
17354 predicate(UseAVX > 0);
17355 match(Set dst (AddD src1 (LoadD src2)));
17356
17357 format %{ "vaddsd $dst, $src1, $src2" %}
17358 ins_cost(150);
17359 ins_encode %{
17360 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17361 %}
17362 ins_pipe(pipe_slow);
17363 %}
17364
17365 instruct addD_reg_imm(regD dst, regD src, immD con) %{
17366 predicate(UseAVX > 0);
17367 match(Set dst (AddD src con));
17368
17369 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17370 ins_cost(150);
17371 ins_encode %{
17372 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17373 %}
17374 ins_pipe(pipe_slow);
17375 %}
17376
17377 instruct subF_reg(regF dst, regF src) %{
17378 predicate(UseAVX == 0);
17379 match(Set dst (SubF dst src));
17380
17381 format %{ "subss $dst, $src" %}
17382 ins_cost(150);
17383 ins_encode %{
17384 __ subss($dst$$XMMRegister, $src$$XMMRegister);
17385 %}
17386 ins_pipe(pipe_slow);
17387 %}
17388
17389 instruct subF_mem(regF dst, memory src) %{
17390 predicate(UseAVX == 0);
17391 match(Set dst (SubF dst (LoadF src)));
17392
17393 format %{ "subss $dst, $src" %}
17394 ins_cost(150);
17395 ins_encode %{
17396 __ subss($dst$$XMMRegister, $src$$Address);
17397 %}
17398 ins_pipe(pipe_slow);
17399 %}
17400
17401 instruct subF_imm(regF dst, immF con) %{
17402 predicate(UseAVX == 0);
17403 match(Set dst (SubF dst con));
17404 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17405 ins_cost(150);
17406 ins_encode %{
17407 __ subss($dst$$XMMRegister, $constantaddress($con));
17408 %}
17409 ins_pipe(pipe_slow);
17410 %}
17411
17412 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
17413 predicate(UseAVX > 0);
17414 match(Set dst (SubF src1 src2));
17415
17416 format %{ "vsubss $dst, $src1, $src2" %}
17417 ins_cost(150);
17418 ins_encode %{
17419 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17420 %}
17421 ins_pipe(pipe_slow);
17422 %}
17423
17424 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
17425 predicate(UseAVX > 0);
17426 match(Set dst (SubF src1 (LoadF src2)));
17427
17428 format %{ "vsubss $dst, $src1, $src2" %}
17429 ins_cost(150);
17430 ins_encode %{
17431 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17432 %}
17433 ins_pipe(pipe_slow);
17434 %}
17435
17436 instruct subF_reg_imm(regF dst, regF src, immF con) %{
17437 predicate(UseAVX > 0);
17438 match(Set dst (SubF src con));
17439
17440 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17441 ins_cost(150);
17442 ins_encode %{
17443 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17444 %}
17445 ins_pipe(pipe_slow);
17446 %}
17447
17448 instruct subD_reg(regD dst, regD src) %{
17449 predicate(UseAVX == 0);
17450 match(Set dst (SubD dst src));
17451
17452 format %{ "subsd $dst, $src" %}
17453 ins_cost(150);
17454 ins_encode %{
17455 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
17456 %}
17457 ins_pipe(pipe_slow);
17458 %}
17459
17460 instruct subD_mem(regD dst, memory src) %{
17461 predicate(UseAVX == 0);
17462 match(Set dst (SubD dst (LoadD src)));
17463
17464 format %{ "subsd $dst, $src" %}
17465 ins_cost(150);
17466 ins_encode %{
17467 __ subsd($dst$$XMMRegister, $src$$Address);
17468 %}
17469 ins_pipe(pipe_slow);
17470 %}
17471
17472 instruct subD_imm(regD dst, immD con) %{
17473 predicate(UseAVX == 0);
17474 match(Set dst (SubD dst con));
17475 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17476 ins_cost(150);
17477 ins_encode %{
17478 __ subsd($dst$$XMMRegister, $constantaddress($con));
17479 %}
17480 ins_pipe(pipe_slow);
17481 %}
17482
17483 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
17484 predicate(UseAVX > 0);
17485 match(Set dst (SubD src1 src2));
17486
17487 format %{ "vsubsd $dst, $src1, $src2" %}
17488 ins_cost(150);
17489 ins_encode %{
17490 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17491 %}
17492 ins_pipe(pipe_slow);
17493 %}
17494
17495 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
17496 predicate(UseAVX > 0);
17497 match(Set dst (SubD src1 (LoadD src2)));
17498
17499 format %{ "vsubsd $dst, $src1, $src2" %}
17500 ins_cost(150);
17501 ins_encode %{
17502 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17503 %}
17504 ins_pipe(pipe_slow);
17505 %}
17506
17507 instruct subD_reg_imm(regD dst, regD src, immD con) %{
17508 predicate(UseAVX > 0);
17509 match(Set dst (SubD src con));
17510
17511 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17512 ins_cost(150);
17513 ins_encode %{
17514 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17515 %}
17516 ins_pipe(pipe_slow);
17517 %}
17518
17519 instruct mulF_reg(regF dst, regF src) %{
17520 predicate(UseAVX == 0);
17521 match(Set dst (MulF dst src));
17522
17523 format %{ "mulss $dst, $src" %}
17524 ins_cost(150);
17525 ins_encode %{
17526 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
17527 %}
17528 ins_pipe(pipe_slow);
17529 %}
17530
17531 instruct mulF_mem(regF dst, memory src) %{
17532 predicate(UseAVX == 0);
17533 match(Set dst (MulF dst (LoadF src)));
17534
17535 format %{ "mulss $dst, $src" %}
17536 ins_cost(150);
17537 ins_encode %{
17538 __ mulss($dst$$XMMRegister, $src$$Address);
17539 %}
17540 ins_pipe(pipe_slow);
17541 %}
17542
17543 instruct mulF_imm(regF dst, immF con) %{
17544 predicate(UseAVX == 0);
17545 match(Set dst (MulF dst con));
17546 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17547 ins_cost(150);
17548 ins_encode %{
17549 __ mulss($dst$$XMMRegister, $constantaddress($con));
17550 %}
17551 ins_pipe(pipe_slow);
17552 %}
17553
17554 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
17555 predicate(UseAVX > 0);
17556 match(Set dst (MulF src1 src2));
17557
17558 format %{ "vmulss $dst, $src1, $src2" %}
17559 ins_cost(150);
17560 ins_encode %{
17561 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17562 %}
17563 ins_pipe(pipe_slow);
17564 %}
17565
17566 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
17567 predicate(UseAVX > 0);
17568 match(Set dst (MulF src1 (LoadF src2)));
17569
17570 format %{ "vmulss $dst, $src1, $src2" %}
17571 ins_cost(150);
17572 ins_encode %{
17573 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17574 %}
17575 ins_pipe(pipe_slow);
17576 %}
17577
17578 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
17579 predicate(UseAVX > 0);
17580 match(Set dst (MulF src con));
17581
17582 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17583 ins_cost(150);
17584 ins_encode %{
17585 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17586 %}
17587 ins_pipe(pipe_slow);
17588 %}
17589
17590 instruct mulD_reg(regD dst, regD src) %{
17591 predicate(UseAVX == 0);
17592 match(Set dst (MulD dst src));
17593
17594 format %{ "mulsd $dst, $src" %}
17595 ins_cost(150);
17596 ins_encode %{
17597 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
17598 %}
17599 ins_pipe(pipe_slow);
17600 %}
17601
17602 instruct mulD_mem(regD dst, memory src) %{
17603 predicate(UseAVX == 0);
17604 match(Set dst (MulD dst (LoadD src)));
17605
17606 format %{ "mulsd $dst, $src" %}
17607 ins_cost(150);
17608 ins_encode %{
17609 __ mulsd($dst$$XMMRegister, $src$$Address);
17610 %}
17611 ins_pipe(pipe_slow);
17612 %}
17613
17614 instruct mulD_imm(regD dst, immD con) %{
17615 predicate(UseAVX == 0);
17616 match(Set dst (MulD dst con));
17617 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17618 ins_cost(150);
17619 ins_encode %{
17620 __ mulsd($dst$$XMMRegister, $constantaddress($con));
17621 %}
17622 ins_pipe(pipe_slow);
17623 %}
17624
17625 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
17626 predicate(UseAVX > 0);
17627 match(Set dst (MulD src1 src2));
17628
17629 format %{ "vmulsd $dst, $src1, $src2" %}
17630 ins_cost(150);
17631 ins_encode %{
17632 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17633 %}
17634 ins_pipe(pipe_slow);
17635 %}
17636
17637 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
17638 predicate(UseAVX > 0);
17639 match(Set dst (MulD src1 (LoadD src2)));
17640
17641 format %{ "vmulsd $dst, $src1, $src2" %}
17642 ins_cost(150);
17643 ins_encode %{
17644 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17645 %}
17646 ins_pipe(pipe_slow);
17647 %}
17648
17649 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
17650 predicate(UseAVX > 0);
17651 match(Set dst (MulD src con));
17652
17653 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17654 ins_cost(150);
17655 ins_encode %{
17656 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17657 %}
17658 ins_pipe(pipe_slow);
17659 %}
17660
17661 instruct divF_reg(regF dst, regF src) %{
17662 predicate(UseAVX == 0);
17663 match(Set dst (DivF dst src));
17664
17665 format %{ "divss $dst, $src" %}
17666 ins_cost(150);
17667 ins_encode %{
17668 __ divss($dst$$XMMRegister, $src$$XMMRegister);
17669 %}
17670 ins_pipe(pipe_slow);
17671 %}
17672
17673 instruct divF_mem(regF dst, memory src) %{
17674 predicate(UseAVX == 0);
17675 match(Set dst (DivF dst (LoadF src)));
17676
17677 format %{ "divss $dst, $src" %}
17678 ins_cost(150);
17679 ins_encode %{
17680 __ divss($dst$$XMMRegister, $src$$Address);
17681 %}
17682 ins_pipe(pipe_slow);
17683 %}
17684
17685 instruct divF_imm(regF dst, immF con) %{
17686 predicate(UseAVX == 0);
17687 match(Set dst (DivF dst con));
17688 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17689 ins_cost(150);
17690 ins_encode %{
17691 __ divss($dst$$XMMRegister, $constantaddress($con));
17692 %}
17693 ins_pipe(pipe_slow);
17694 %}
17695
17696 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
17697 predicate(UseAVX > 0);
17698 match(Set dst (DivF src1 src2));
17699
17700 format %{ "vdivss $dst, $src1, $src2" %}
17701 ins_cost(150);
17702 ins_encode %{
17703 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17704 %}
17705 ins_pipe(pipe_slow);
17706 %}
17707
17708 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
17709 predicate(UseAVX > 0);
17710 match(Set dst (DivF src1 (LoadF src2)));
17711
17712 format %{ "vdivss $dst, $src1, $src2" %}
17713 ins_cost(150);
17714 ins_encode %{
17715 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17716 %}
17717 ins_pipe(pipe_slow);
17718 %}
17719
17720 instruct divF_reg_imm(regF dst, regF src, immF con) %{
17721 predicate(UseAVX > 0);
17722 match(Set dst (DivF src con));
17723
17724 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17725 ins_cost(150);
17726 ins_encode %{
17727 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17728 %}
17729 ins_pipe(pipe_slow);
17730 %}
17731
17732 instruct divD_reg(regD dst, regD src) %{
17733 predicate(UseAVX == 0);
17734 match(Set dst (DivD dst src));
17735
17736 format %{ "divsd $dst, $src" %}
17737 ins_cost(150);
17738 ins_encode %{
17739 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
17740 %}
17741 ins_pipe(pipe_slow);
17742 %}
17743
17744 instruct divD_mem(regD dst, memory src) %{
17745 predicate(UseAVX == 0);
17746 match(Set dst (DivD dst (LoadD src)));
17747
17748 format %{ "divsd $dst, $src" %}
17749 ins_cost(150);
17750 ins_encode %{
17751 __ divsd($dst$$XMMRegister, $src$$Address);
17752 %}
17753 ins_pipe(pipe_slow);
17754 %}
17755
17756 instruct divD_imm(regD dst, immD con) %{
17757 predicate(UseAVX == 0);
17758 match(Set dst (DivD dst con));
17759 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17760 ins_cost(150);
17761 ins_encode %{
17762 __ divsd($dst$$XMMRegister, $constantaddress($con));
17763 %}
17764 ins_pipe(pipe_slow);
17765 %}
17766
17767 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
17768 predicate(UseAVX > 0);
17769 match(Set dst (DivD src1 src2));
17770
17771 format %{ "vdivsd $dst, $src1, $src2" %}
17772 ins_cost(150);
17773 ins_encode %{
17774 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17775 %}
17776 ins_pipe(pipe_slow);
17777 %}
17778
17779 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
17780 predicate(UseAVX > 0);
17781 match(Set dst (DivD src1 (LoadD src2)));
17782
17783 format %{ "vdivsd $dst, $src1, $src2" %}
17784 ins_cost(150);
17785 ins_encode %{
17786 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17787 %}
17788 ins_pipe(pipe_slow);
17789 %}
17790
17791 instruct divD_reg_imm(regD dst, regD src, immD con) %{
17792 predicate(UseAVX > 0);
17793 match(Set dst (DivD src con));
17794
17795 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17796 ins_cost(150);
17797 ins_encode %{
17798 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17799 %}
17800 ins_pipe(pipe_slow);
17801 %}
17802
17803 instruct absF_reg(regF dst) %{
17804 predicate(UseAVX == 0);
17805 match(Set dst (AbsF dst));
17806 ins_cost(150);
17807 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
17808 ins_encode %{
17809 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
17810 %}
17811 ins_pipe(pipe_slow);
17812 %}
17813
17814 instruct absF_reg_reg(vlRegF dst, vlRegF src) %{
17815 predicate(UseAVX > 0);
17816 match(Set dst (AbsF src));
17817 ins_cost(150);
17818 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
17819 ins_encode %{
17820 int vlen_enc = Assembler::AVX_128bit;
17821 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
17822 ExternalAddress(float_signmask()), vlen_enc);
17823 %}
17824 ins_pipe(pipe_slow);
17825 %}
17826
17827 instruct absD_reg(regD dst) %{
17828 predicate(UseAVX == 0);
17829 match(Set dst (AbsD dst));
17830 ins_cost(150);
17831 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
17832 "# abs double by sign masking" %}
17833 ins_encode %{
17834 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
17835 %}
17836 ins_pipe(pipe_slow);
17837 %}
17838
17839 instruct absD_reg_reg(vlRegD dst, vlRegD src) %{
17840 predicate(UseAVX > 0);
17841 match(Set dst (AbsD src));
17842 ins_cost(150);
17843 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
17844 "# abs double by sign masking" %}
17845 ins_encode %{
17846 int vlen_enc = Assembler::AVX_128bit;
17847 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
17848 ExternalAddress(double_signmask()), vlen_enc);
17849 %}
17850 ins_pipe(pipe_slow);
17851 %}
17852
17853 instruct negF_reg(regF dst) %{
17854 predicate(UseAVX == 0);
17855 match(Set dst (NegF dst));
17856 ins_cost(150);
17857 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
17858 ins_encode %{
17859 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
17860 %}
17861 ins_pipe(pipe_slow);
17862 %}
17863
17864 instruct negF_reg_reg(vlRegF dst, vlRegF src) %{
17865 predicate(UseAVX > 0);
17866 match(Set dst (NegF src));
17867 ins_cost(150);
17868 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
17869 ins_encode %{
17870 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
17871 ExternalAddress(float_signflip()));
17872 %}
17873 ins_pipe(pipe_slow);
17874 %}
17875
17876 instruct negD_reg(regD dst) %{
17877 predicate(UseAVX == 0);
17878 match(Set dst (NegD dst));
17879 ins_cost(150);
17880 format %{ "xorpd $dst, [0x8000000000000000]\t"
17881 "# neg double by sign flipping" %}
17882 ins_encode %{
17883 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
17884 %}
17885 ins_pipe(pipe_slow);
17886 %}
17887
17888 instruct negD_reg_reg(vlRegD dst, vlRegD src) %{
17889 predicate(UseAVX > 0);
17890 match(Set dst (NegD src));
17891 ins_cost(150);
17892 format %{ "vnegatesd $dst, $src, [0x8000000000000000]\t"
17893 "# neg double by sign flipping" %}
17894 ins_encode %{
17895 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
17896 ExternalAddress(double_signflip()));
17897 %}
17898 ins_pipe(pipe_slow);
17899 %}
17900
17901 // sqrtss instruction needs destination register to be pre initialized for best performance
17902 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17903 instruct sqrtF_reg(regF dst) %{
17904 match(Set dst (SqrtF dst));
17905 format %{ "sqrtss $dst, $dst" %}
17906 ins_encode %{
17907 __ sqrtss($dst$$XMMRegister, $dst$$XMMRegister);
17908 %}
17909 ins_pipe(pipe_slow);
17910 %}
17911
17912 // sqrtsd instruction needs destination register to be pre initialized for best performance
17913 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17914 instruct sqrtD_reg(regD dst) %{
17915 match(Set dst (SqrtD dst));
17916 format %{ "sqrtsd $dst, $dst" %}
17917 ins_encode %{
17918 __ sqrtsd($dst$$XMMRegister, $dst$$XMMRegister);
17919 %}
17920 ins_pipe(pipe_slow);
17921 %}
17922
17923 instruct convF2HF_reg_reg(rRegI dst, vlRegF src, vlRegF tmp) %{
17924 effect(TEMP tmp);
17925 match(Set dst (ConvF2HF src));
17926 ins_cost(125);
17927 format %{ "vcvtps2ph $dst,$src \t using $tmp as TEMP"%}
17928 ins_encode %{
17929 __ flt_to_flt16($dst$$Register, $src$$XMMRegister, $tmp$$XMMRegister);
17930 %}
17931 ins_pipe( pipe_slow );
17932 %}
17933
17934 instruct convF2HF_mem_reg(memory mem, regF src, kReg ktmp, rRegI rtmp) %{
17935 predicate((UseAVX > 2) && VM_Version::supports_avx512vl());
17936 effect(TEMP ktmp, TEMP rtmp);
17937 match(Set mem (StoreC mem (ConvF2HF src)));
17938 format %{ "evcvtps2ph $mem,$src \t using $ktmp and $rtmp as TEMP" %}
17939 ins_encode %{
17940 __ movl($rtmp$$Register, 0x1);
17941 __ kmovwl($ktmp$$KRegister, $rtmp$$Register);
17942 __ evcvtps2ph($mem$$Address, $ktmp$$KRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
17943 %}
17944 ins_pipe( pipe_slow );
17945 %}
17946
17947 instruct vconvF2HF(vec dst, vec src) %{
17948 match(Set dst (VectorCastF2HF src));
17949 format %{ "vector_conv_F2HF $dst $src" %}
17950 ins_encode %{
17951 int vlen_enc = vector_length_encoding(this, $src);
17952 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, vlen_enc);
17953 %}
17954 ins_pipe( pipe_slow );
17955 %}
17956
17957 instruct vconvF2HF_mem_reg(memory mem, vec src) %{
17958 predicate(n->as_StoreVector()->memory_size() >= 16);
17959 match(Set mem (StoreVector mem (VectorCastF2HF src)));
17960 format %{ "vcvtps2ph $mem,$src" %}
17961 ins_encode %{
17962 int vlen_enc = vector_length_encoding(this, $src);
17963 __ vcvtps2ph($mem$$Address, $src$$XMMRegister, 0x04, vlen_enc);
17964 %}
17965 ins_pipe( pipe_slow );
17966 %}
17967
17968 instruct convHF2F_reg_reg(vlRegF dst, rRegI src) %{
17969 match(Set dst (ConvHF2F src));
17970 format %{ "vcvtph2ps $dst,$src" %}
17971 ins_encode %{
17972 __ flt16_to_flt($dst$$XMMRegister, $src$$Register);
17973 %}
17974 ins_pipe( pipe_slow );
17975 %}
17976
17977 instruct vconvHF2F_reg_mem(vec dst, memory mem) %{
17978 match(Set dst (VectorCastHF2F (LoadVector mem)));
17979 format %{ "vcvtph2ps $dst,$mem" %}
17980 ins_encode %{
17981 int vlen_enc = vector_length_encoding(this);
17982 __ vcvtph2ps($dst$$XMMRegister, $mem$$Address, vlen_enc);
17983 %}
17984 ins_pipe( pipe_slow );
17985 %}
17986
17987 instruct vconvHF2F(vec dst, vec src) %{
17988 match(Set dst (VectorCastHF2F src));
17989 ins_cost(125);
17990 format %{ "vector_conv_HF2F $dst,$src" %}
17991 ins_encode %{
17992 int vlen_enc = vector_length_encoding(this);
17993 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
17994 %}
17995 ins_pipe( pipe_slow );
17996 %}
17997
17998 // ---------------------------------------- VectorReinterpret ------------------------------------
17999 instruct reinterpret_mask(kReg dst) %{
18000 predicate(n->bottom_type()->isa_pvectmask() &&
18001 Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
18002 match(Set dst (VectorReinterpret dst));
18003 ins_cost(125);
18004 format %{ "vector_reinterpret $dst\t!" %}
18005 ins_encode %{
18006 // empty
18007 %}
18008 ins_pipe( pipe_slow );
18009 %}
18010
18011 instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
18012 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18013 n->bottom_type()->isa_pvectmask() &&
18014 n->in(1)->bottom_type()->isa_pvectmask() &&
18015 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
18016 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18017 match(Set dst (VectorReinterpret src));
18018 effect(TEMP xtmp);
18019 format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
18020 ins_encode %{
18021 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_SHORT);
18022 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18023 assert(src_sz == dst_sz , "src and dst size mismatch");
18024 int vlen_enc = vector_length_encoding(src_sz);
18025 __ evpmovm2w($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18026 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18027 %}
18028 ins_pipe( pipe_slow );
18029 %}
18030
18031 instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
18032 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18033 n->bottom_type()->isa_pvectmask() &&
18034 n->in(1)->bottom_type()->isa_pvectmask() &&
18035 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
18036 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
18037 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18038 match(Set dst (VectorReinterpret src));
18039 effect(TEMP xtmp);
18040 format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
18041 ins_encode %{
18042 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_INT);
18043 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18044 assert(src_sz == dst_sz , "src and dst size mismatch");
18045 int vlen_enc = vector_length_encoding(src_sz);
18046 __ evpmovm2d($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18047 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18048 %}
18049 ins_pipe( pipe_slow );
18050 %}
18051
18052 instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
18053 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18054 n->bottom_type()->isa_pvectmask() &&
18055 n->in(1)->bottom_type()->isa_pvectmask() &&
18056 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
18057 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
18058 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18059 match(Set dst (VectorReinterpret src));
18060 effect(TEMP xtmp);
18061 format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
18062 ins_encode %{
18063 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_LONG);
18064 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18065 assert(src_sz == dst_sz , "src and dst size mismatch");
18066 int vlen_enc = vector_length_encoding(src_sz);
18067 __ evpmovm2q($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18068 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18069 %}
18070 ins_pipe( pipe_slow );
18071 %}
18072
18073 instruct reinterpret(vec dst) %{
18074 predicate(!n->bottom_type()->isa_pvectmask() &&
18075 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
18076 match(Set dst (VectorReinterpret dst));
18077 ins_cost(125);
18078 format %{ "vector_reinterpret $dst\t!" %}
18079 ins_encode %{
18080 // empty
18081 %}
18082 ins_pipe( pipe_slow );
18083 %}
18084
18085 instruct reinterpret_expand(vec dst, vec src) %{
18086 predicate(UseAVX == 0 &&
18087 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18088 match(Set dst (VectorReinterpret src));
18089 ins_cost(125);
18090 effect(TEMP dst);
18091 format %{ "vector_reinterpret_expand $dst,$src" %}
18092 ins_encode %{
18093 assert(Matcher::vector_length_in_bytes(this) <= 16, "required");
18094 assert(Matcher::vector_length_in_bytes(this, $src) <= 8, "required");
18095
18096 int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
18097 if (src_vlen_in_bytes == 4) {
18098 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), noreg);
18099 } else {
18100 assert(src_vlen_in_bytes == 8, "");
18101 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), noreg);
18102 }
18103 __ pand($dst$$XMMRegister, $src$$XMMRegister);
18104 %}
18105 ins_pipe( pipe_slow );
18106 %}
18107
18108 instruct vreinterpret_expand4(legVec dst, vec src) %{
18109 predicate(UseAVX > 0 &&
18110 !n->bottom_type()->isa_pvectmask() &&
18111 (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
18112 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18113 match(Set dst (VectorReinterpret src));
18114 ins_cost(125);
18115 format %{ "vector_reinterpret_expand $dst,$src" %}
18116 ins_encode %{
18117 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), 0, noreg);
18118 %}
18119 ins_pipe( pipe_slow );
18120 %}
18121
18122
18123 instruct vreinterpret_expand(legVec dst, vec src) %{
18124 predicate(UseAVX > 0 &&
18125 !n->bottom_type()->isa_pvectmask() &&
18126 (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
18127 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18128 match(Set dst (VectorReinterpret src));
18129 ins_cost(125);
18130 format %{ "vector_reinterpret_expand $dst,$src\t!" %}
18131 ins_encode %{
18132 switch (Matcher::vector_length_in_bytes(this, $src)) {
18133 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
18134 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
18135 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
18136 default: ShouldNotReachHere();
18137 }
18138 %}
18139 ins_pipe( pipe_slow );
18140 %}
18141
18142 instruct reinterpret_shrink(vec dst, legVec src) %{
18143 predicate(!n->bottom_type()->isa_pvectmask() &&
18144 Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
18145 match(Set dst (VectorReinterpret src));
18146 ins_cost(125);
18147 format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
18148 ins_encode %{
18149 switch (Matcher::vector_length_in_bytes(this)) {
18150 case 4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
18151 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
18152 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
18153 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
18154 default: ShouldNotReachHere();
18155 }
18156 %}
18157 ins_pipe( pipe_slow );
18158 %}
18159
18160 // ----------------------------------------------------------------------------------------------------
18161
18162 instruct roundD_reg(legRegD dst, legRegD src, immU8 rmode) %{
18163 match(Set dst (RoundDoubleMode src rmode));
18164 format %{ "roundsd $dst,$src" %}
18165 ins_cost(150);
18166 ins_encode %{
18167 assert(UseSSE >= 4, "required");
18168 if ((UseAVX == 0) && ($dst$$XMMRegister != $src$$XMMRegister)) {
18169 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18170 }
18171 __ roundsd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant);
18172 %}
18173 ins_pipe(pipe_slow);
18174 %}
18175
18176 instruct roundD_imm(legRegD dst, immD con, immU8 rmode) %{
18177 match(Set dst (RoundDoubleMode con rmode));
18178 format %{ "roundsd $dst,[$constantaddress]\t# load from constant table: double=$con" %}
18179 ins_cost(150);
18180 ins_encode %{
18181 assert(UseSSE >= 4, "required");
18182 __ roundsd($dst$$XMMRegister, $constantaddress($con), $rmode$$constant, noreg);
18183 %}
18184 ins_pipe(pipe_slow);
18185 %}
18186
18187 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
18188 predicate(Matcher::vector_length(n) < 8);
18189 match(Set dst (RoundDoubleModeV src rmode));
18190 format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
18191 ins_encode %{
18192 assert(UseAVX > 0, "required");
18193 int vlen_enc = vector_length_encoding(this);
18194 __ vroundpd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, vlen_enc);
18195 %}
18196 ins_pipe( pipe_slow );
18197 %}
18198
18199 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
18200 predicate(Matcher::vector_length(n) == 8);
18201 match(Set dst (RoundDoubleModeV src rmode));
18202 format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
18203 ins_encode %{
18204 assert(UseAVX > 2, "required");
18205 __ vrndscalepd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, Assembler::AVX_512bit);
18206 %}
18207 ins_pipe( pipe_slow );
18208 %}
18209
18210 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
18211 predicate(Matcher::vector_length(n) < 8);
18212 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
18213 format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
18214 ins_encode %{
18215 assert(UseAVX > 0, "required");
18216 int vlen_enc = vector_length_encoding(this);
18217 __ vroundpd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, vlen_enc);
18218 %}
18219 ins_pipe( pipe_slow );
18220 %}
18221
18222 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
18223 predicate(Matcher::vector_length(n) == 8);
18224 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
18225 format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
18226 ins_encode %{
18227 assert(UseAVX > 2, "required");
18228 __ vrndscalepd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, Assembler::AVX_512bit);
18229 %}
18230 ins_pipe( pipe_slow );
18231 %}
18232
18233 instruct onspinwait() %{
18234 match(OnSpinWait);
18235 ins_cost(200);
18236
18237 format %{
18238 $$template
18239 $$emit$$"pause\t! membar_onspinwait"
18240 %}
18241 ins_encode %{
18242 __ pause();
18243 %}
18244 ins_pipe(pipe_slow);
18245 %}
18246
18247 // a * b + c
18248 instruct fmaD_reg(regD a, regD b, regD c) %{
18249 match(Set c (FmaD c (Binary a b)));
18250 format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
18251 ins_cost(150);
18252 ins_encode %{
18253 assert(UseFMA, "Needs FMA instructions support.");
18254 __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
18255 %}
18256 ins_pipe( pipe_slow );
18257 %}
18258
18259 // a * b + c
18260 instruct fmaF_reg(regF a, regF b, regF c) %{
18261 match(Set c (FmaF c (Binary a b)));
18262 format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
18263 ins_cost(150);
18264 ins_encode %{
18265 assert(UseFMA, "Needs FMA instructions support.");
18266 __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
18267 %}
18268 ins_pipe( pipe_slow );
18269 %}
18270
18271 // ====================VECTOR INSTRUCTIONS=====================================
18272
18273 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
18274 instruct MoveVec2Leg(legVec dst, vec src) %{
18275 match(Set dst src);
18276 format %{ "" %}
18277 ins_encode %{
18278 ShouldNotReachHere();
18279 %}
18280 ins_pipe( fpu_reg_reg );
18281 %}
18282
18283 instruct MoveLeg2Vec(vec dst, legVec src) %{
18284 match(Set dst src);
18285 format %{ "" %}
18286 ins_encode %{
18287 ShouldNotReachHere();
18288 %}
18289 ins_pipe( fpu_reg_reg );
18290 %}
18291
18292 // ============================================================================
18293
18294 // Load vectors generic operand pattern
18295 instruct loadV(vec dst, memory mem) %{
18296 match(Set dst (LoadVector mem));
18297 ins_cost(125);
18298 format %{ "load_vector $dst,$mem" %}
18299 ins_encode %{
18300 BasicType bt = Matcher::vector_element_basic_type(this);
18301 __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this));
18302 %}
18303 ins_pipe( pipe_slow );
18304 %}
18305
18306 // Store vectors generic operand pattern.
18307 instruct storeV(memory mem, vec src) %{
18308 match(Set mem (StoreVector mem src));
18309 ins_cost(145);
18310 format %{ "store_vector $mem,$src\n\t" %}
18311 ins_encode %{
18312 switch (Matcher::vector_length_in_bytes(this, $src)) {
18313 case 4: __ movdl ($mem$$Address, $src$$XMMRegister); break;
18314 case 8: __ movq ($mem$$Address, $src$$XMMRegister); break;
18315 case 16: __ movdqu ($mem$$Address, $src$$XMMRegister); break;
18316 case 32: __ vmovdqu ($mem$$Address, $src$$XMMRegister); break;
18317 case 64: __ evmovdqul($mem$$Address, $src$$XMMRegister, Assembler::AVX_512bit); break;
18318 default: ShouldNotReachHere();
18319 }
18320 %}
18321 ins_pipe( pipe_slow );
18322 %}
18323
18324 // ---------------------------------------- Gather ------------------------------------
18325
18326 // Gather BYTE, SHORT, INT, LONG, FLOAT, DOUBLE
18327
18328 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
18329 predicate(!VM_Version::supports_avx512vl() && !is_subword_type(Matcher::vector_element_basic_type(n)) &&
18330 Matcher::vector_length_in_bytes(n) <= 32);
18331 match(Set dst (LoadVectorGather mem idx));
18332 effect(TEMP dst, TEMP tmp, TEMP mask);
18333 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
18334 ins_encode %{
18335 int vlen_enc = vector_length_encoding(this);
18336 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18337 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18338 __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18339 __ lea($tmp$$Register, $mem$$Address);
18340 __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18341 %}
18342 ins_pipe( pipe_slow );
18343 %}
18344
18345
18346 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
18347 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18348 !is_subword_type(Matcher::vector_element_basic_type(n)));
18349 match(Set dst (LoadVectorGather mem idx));
18350 effect(TEMP dst, TEMP tmp, TEMP ktmp);
18351 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
18352 ins_encode %{
18353 int vlen_enc = vector_length_encoding(this);
18354 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18355 __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
18356 __ lea($tmp$$Register, $mem$$Address);
18357 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18358 %}
18359 ins_pipe( pipe_slow );
18360 %}
18361
18362 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18363 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18364 !is_subword_type(Matcher::vector_element_basic_type(n)));
18365 match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
18366 effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
18367 format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
18368 ins_encode %{
18369 assert(UseAVX > 2, "sanity");
18370 int vlen_enc = vector_length_encoding(this);
18371 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18372 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18373 // Note: Since gather instruction partially updates the opmask register used
18374 // for predication hense moving mask operand to a temporary.
18375 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18376 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18377 __ lea($tmp$$Register, $mem$$Address);
18378 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18379 %}
18380 ins_pipe( pipe_slow );
18381 %}
18382
18383 instruct vgather_subwordLE8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegI rtmp) %{
18384 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18385 match(Set dst (LoadVectorGather mem idx_base));
18386 effect(TEMP tmp, TEMP rtmp);
18387 format %{ "vector_gatherLE8 $dst, $mem, $idx_base\t! using $tmp and $rtmp as TEMP" %}
18388 ins_encode %{
18389 int vlen_enc = vector_length_encoding(this);
18390 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18391 __ lea($tmp$$Register, $mem$$Address);
18392 __ vgather8b(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp$$Register, vlen_enc);
18393 %}
18394 ins_pipe( pipe_slow );
18395 %}
18396
18397 instruct vgather_subwordGT8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegP idx_base_temp,
18398 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI length, rFlagsReg cr) %{
18399 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18400 match(Set dst (LoadVectorGather mem idx_base));
18401 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP length, KILL cr);
18402 format %{ "vector_gatherGT8 $dst, $mem, $idx_base\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp and $length as TEMP" %}
18403 ins_encode %{
18404 int vlen_enc = vector_length_encoding(this);
18405 int vector_len = Matcher::vector_length(this);
18406 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18407 __ lea($tmp$$Register, $mem$$Address);
18408 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18409 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, noreg, $xtmp1$$XMMRegister,
18410 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, noreg, $length$$Register, vector_len, vlen_enc);
18411 %}
18412 ins_pipe( pipe_slow );
18413 %}
18414
18415 instruct vgather_masked_subwordLE8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegL mask_idx, rRegP tmp, rRegI rtmp, rRegL rtmp2, rFlagsReg cr) %{
18416 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18417 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18418 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18419 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18420 ins_encode %{
18421 int vlen_enc = vector_length_encoding(this);
18422 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18423 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18424 __ lea($tmp$$Register, $mem$$Address);
18425 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18426 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18427 %}
18428 ins_pipe( pipe_slow );
18429 %}
18430
18431 instruct vgather_masked_subwordGT8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegP tmp, rRegP idx_base_temp,
18432 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegL rtmp2, rRegL mask_idx, rRegI length, rFlagsReg cr) %{
18433 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18434 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18435 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18436 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18437 ins_encode %{
18438 int vlen_enc = vector_length_encoding(this);
18439 int vector_len = Matcher::vector_length(this);
18440 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18441 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18442 __ lea($tmp$$Register, $mem$$Address);
18443 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18444 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18445 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18446 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18447 %}
18448 ins_pipe( pipe_slow );
18449 %}
18450
18451 instruct vgather_masked_subwordLE8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegI mask_idx, rRegP tmp, rRegI rtmp, rRegI rtmp2, rFlagsReg cr) %{
18452 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18453 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18454 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18455 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18456 ins_encode %{
18457 int vlen_enc = vector_length_encoding(this);
18458 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18459 __ lea($tmp$$Register, $mem$$Address);
18460 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18461 if (elem_bt == T_SHORT) {
18462 __ movl($mask_idx$$Register, 0x55555555);
18463 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18464 }
18465 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18466 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18467 %}
18468 ins_pipe( pipe_slow );
18469 %}
18470
18471 instruct vgather_masked_subwordGT8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegP tmp, rRegP idx_base_temp,
18472 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI rtmp2, rRegI mask_idx, rRegI length, rFlagsReg cr) %{
18473 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18474 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18475 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18476 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18477 ins_encode %{
18478 int vlen_enc = vector_length_encoding(this);
18479 int vector_len = Matcher::vector_length(this);
18480 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18481 __ lea($tmp$$Register, $mem$$Address);
18482 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18483 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18484 if (elem_bt == T_SHORT) {
18485 __ movl($mask_idx$$Register, 0x55555555);
18486 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18487 }
18488 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18489 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18490 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18491 %}
18492 ins_pipe( pipe_slow );
18493 %}
18494
18495 // ====================Scatter=======================================
18496
18497 // Scatter INT, LONG, FLOAT, DOUBLE
18498
18499 instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
18500 predicate(UseAVX > 2);
18501 match(Set mem (StoreVectorScatter mem (Binary src idx)));
18502 effect(TEMP tmp, TEMP ktmp);
18503 format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
18504 ins_encode %{
18505 int vlen_enc = vector_length_encoding(this, $src);
18506 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18507
18508 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18509 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18510
18511 __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
18512 __ lea($tmp$$Register, $mem$$Address);
18513 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18514 %}
18515 ins_pipe( pipe_slow );
18516 %}
18517
18518 instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18519 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx mask))));
18520 effect(TEMP tmp, TEMP ktmp);
18521 format %{ "store_vector_scatter_masked $mem, $idx, $src, $mask\t!" %}
18522 ins_encode %{
18523 int vlen_enc = vector_length_encoding(this, $src);
18524 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18525 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18526 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18527 // Note: Since scatter instruction partially updates the opmask register used
18528 // for predication hense moving mask operand to a temporary.
18529 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18530 __ lea($tmp$$Register, $mem$$Address);
18531 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18532 %}
18533 ins_pipe( pipe_slow );
18534 %}
18535
18536 // ====================REPLICATE=======================================
18537
18538 // Replicate byte scalar to be vector
18539 instruct vReplB_reg(vec dst, rRegI src) %{
18540 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
18541 match(Set dst (Replicate src));
18542 format %{ "replicateB $dst,$src" %}
18543 ins_encode %{
18544 uint vlen = Matcher::vector_length(this);
18545 if (UseAVX >= 2) {
18546 int vlen_enc = vector_length_encoding(this);
18547 if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18548 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
18549 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
18550 } else {
18551 __ movdl($dst$$XMMRegister, $src$$Register);
18552 __ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18553 }
18554 } else {
18555 assert(UseAVX < 2, "");
18556 __ movdl($dst$$XMMRegister, $src$$Register);
18557 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
18558 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18559 if (vlen >= 16) {
18560 assert(vlen == 16, "");
18561 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18562 }
18563 }
18564 %}
18565 ins_pipe( pipe_slow );
18566 %}
18567
18568 instruct ReplB_mem(vec dst, memory mem) %{
18569 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
18570 match(Set dst (Replicate (LoadB mem)));
18571 format %{ "replicateB $dst,$mem" %}
18572 ins_encode %{
18573 int vlen_enc = vector_length_encoding(this);
18574 __ vpbroadcastb($dst$$XMMRegister, $mem$$Address, vlen_enc);
18575 %}
18576 ins_pipe( pipe_slow );
18577 %}
18578
18579 // ====================ReplicateS=======================================
18580
18581 instruct vReplS_reg(vec dst, rRegI src) %{
18582 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
18583 match(Set dst (Replicate src));
18584 format %{ "replicateS $dst,$src" %}
18585 ins_encode %{
18586 uint vlen = Matcher::vector_length(this);
18587 int vlen_enc = vector_length_encoding(this);
18588 if (UseAVX >= 2) {
18589 if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18590 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
18591 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
18592 } else {
18593 __ movdl($dst$$XMMRegister, $src$$Register);
18594 __ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18595 }
18596 } else {
18597 assert(UseAVX < 2, "");
18598 __ movdl($dst$$XMMRegister, $src$$Register);
18599 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18600 if (vlen >= 8) {
18601 assert(vlen == 8, "");
18602 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18603 }
18604 }
18605 %}
18606 ins_pipe( pipe_slow );
18607 %}
18608
18609 instruct ReplHF_imm(vec dst, immH con, rRegI rtmp) %{
18610 match(Set dst (Replicate con));
18611 effect(TEMP rtmp);
18612 format %{ "replicateHF $dst, $con \t! using $rtmp as TEMP" %}
18613 ins_encode %{
18614 int vlen_enc = vector_length_encoding(this);
18615 BasicType bt = Matcher::vector_element_basic_type(this);
18616 assert(VM_Version::supports_avx512_fp16() && bt == T_SHORT, "");
18617 __ movl($rtmp$$Register, $con$$constant);
18618 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18619 %}
18620 ins_pipe( pipe_slow );
18621 %}
18622
18623 instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
18624 predicate(VM_Version::supports_avx512_fp16() && Matcher::vector_element_basic_type(n) == T_SHORT);
18625 match(Set dst (Replicate src));
18626 effect(TEMP rtmp);
18627 format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
18628 ins_encode %{
18629 int vlen_enc = vector_length_encoding(this);
18630 __ evmovw($rtmp$$Register, $src$$XMMRegister);
18631 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18632 %}
18633 ins_pipe( pipe_slow );
18634 %}
18635
18636 instruct ReplS_mem(vec dst, memory mem) %{
18637 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
18638 match(Set dst (Replicate (LoadS mem)));
18639 format %{ "replicateS $dst,$mem" %}
18640 ins_encode %{
18641 int vlen_enc = vector_length_encoding(this);
18642 __ vpbroadcastw($dst$$XMMRegister, $mem$$Address, vlen_enc);
18643 %}
18644 ins_pipe( pipe_slow );
18645 %}
18646
18647 // ====================ReplicateI=======================================
18648
18649 instruct ReplI_reg(vec dst, rRegI src) %{
18650 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18651 match(Set dst (Replicate src));
18652 format %{ "replicateI $dst,$src" %}
18653 ins_encode %{
18654 uint vlen = Matcher::vector_length(this);
18655 int vlen_enc = vector_length_encoding(this);
18656 if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18657 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
18658 } else if (VM_Version::supports_avx2()) {
18659 __ movdl($dst$$XMMRegister, $src$$Register);
18660 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18661 } else {
18662 __ movdl($dst$$XMMRegister, $src$$Register);
18663 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18664 }
18665 %}
18666 ins_pipe( pipe_slow );
18667 %}
18668
18669 instruct ReplI_mem(vec dst, memory mem) %{
18670 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18671 match(Set dst (Replicate (LoadI mem)));
18672 format %{ "replicateI $dst,$mem" %}
18673 ins_encode %{
18674 int vlen_enc = vector_length_encoding(this);
18675 if (VM_Version::supports_avx2()) {
18676 __ vpbroadcastd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18677 } else if (VM_Version::supports_avx()) {
18678 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18679 } else {
18680 __ movdl($dst$$XMMRegister, $mem$$Address);
18681 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18682 }
18683 %}
18684 ins_pipe( pipe_slow );
18685 %}
18686
18687 instruct ReplI_imm(vec dst, immI con) %{
18688 predicate(Matcher::is_non_long_integral_vector(n));
18689 match(Set dst (Replicate con));
18690 format %{ "replicateI $dst,$con" %}
18691 ins_encode %{
18692 InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant,
18693 (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) /
18694 type2aelembytes(Matcher::vector_element_basic_type(this))));
18695 BasicType bt = Matcher::vector_element_basic_type(this);
18696 int vlen = Matcher::vector_length_in_bytes(this);
18697 __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen);
18698 %}
18699 ins_pipe( pipe_slow );
18700 %}
18701
18702 // Replicate scalar zero to be vector
18703 instruct ReplI_zero(vec dst, immI_0 zero) %{
18704 predicate(Matcher::is_non_long_integral_vector(n));
18705 match(Set dst (Replicate zero));
18706 format %{ "replicateI $dst,$zero" %}
18707 ins_encode %{
18708 int vlen_enc = vector_length_encoding(this);
18709 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18710 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18711 } else {
18712 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18713 }
18714 %}
18715 ins_pipe( fpu_reg_reg );
18716 %}
18717
18718 instruct ReplI_M1(vec dst, immI_M1 con) %{
18719 predicate(Matcher::is_non_long_integral_vector(n));
18720 match(Set dst (Replicate con));
18721 format %{ "vallones $dst" %}
18722 ins_encode %{
18723 int vector_len = vector_length_encoding(this);
18724 __ vallones($dst$$XMMRegister, vector_len);
18725 %}
18726 ins_pipe( pipe_slow );
18727 %}
18728
18729 // ====================ReplicateL=======================================
18730
18731 // Replicate long (8 byte) scalar to be vector
18732 instruct ReplL_reg(vec dst, rRegL src) %{
18733 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18734 match(Set dst (Replicate src));
18735 format %{ "replicateL $dst,$src" %}
18736 ins_encode %{
18737 int vlen = Matcher::vector_length(this);
18738 int vlen_enc = vector_length_encoding(this);
18739 if (vlen == 8 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18740 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vlen_enc);
18741 } else if (VM_Version::supports_avx2()) {
18742 __ movdq($dst$$XMMRegister, $src$$Register);
18743 __ vpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18744 } else {
18745 __ movdq($dst$$XMMRegister, $src$$Register);
18746 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18747 }
18748 %}
18749 ins_pipe( pipe_slow );
18750 %}
18751
18752 instruct ReplL_mem(vec dst, memory mem) %{
18753 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18754 match(Set dst (Replicate (LoadL mem)));
18755 format %{ "replicateL $dst,$mem" %}
18756 ins_encode %{
18757 int vlen_enc = vector_length_encoding(this);
18758 if (VM_Version::supports_avx2()) {
18759 __ vpbroadcastq($dst$$XMMRegister, $mem$$Address, vlen_enc);
18760 } else if (VM_Version::supports_sse3()) {
18761 __ movddup($dst$$XMMRegister, $mem$$Address);
18762 } else {
18763 __ movq($dst$$XMMRegister, $mem$$Address);
18764 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18765 }
18766 %}
18767 ins_pipe( pipe_slow );
18768 %}
18769
18770 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
18771 instruct ReplL_imm(vec dst, immL con) %{
18772 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18773 match(Set dst (Replicate con));
18774 format %{ "replicateL $dst,$con" %}
18775 ins_encode %{
18776 InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18777 int vlen = Matcher::vector_length_in_bytes(this);
18778 __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen);
18779 %}
18780 ins_pipe( pipe_slow );
18781 %}
18782
18783 instruct ReplL_zero(vec dst, immL0 zero) %{
18784 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18785 match(Set dst (Replicate zero));
18786 format %{ "replicateL $dst,$zero" %}
18787 ins_encode %{
18788 int vlen_enc = vector_length_encoding(this);
18789 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18790 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18791 } else {
18792 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18793 }
18794 %}
18795 ins_pipe( fpu_reg_reg );
18796 %}
18797
18798 instruct ReplL_M1(vec dst, immL_M1 con) %{
18799 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18800 match(Set dst (Replicate con));
18801 format %{ "vallones $dst" %}
18802 ins_encode %{
18803 int vector_len = vector_length_encoding(this);
18804 __ vallones($dst$$XMMRegister, vector_len);
18805 %}
18806 ins_pipe( pipe_slow );
18807 %}
18808
18809 // ====================ReplicateF=======================================
18810
18811 instruct vReplF_reg(vec dst, vlRegF src) %{
18812 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18813 match(Set dst (Replicate src));
18814 format %{ "replicateF $dst,$src" %}
18815 ins_encode %{
18816 uint vlen = Matcher::vector_length(this);
18817 int vlen_enc = vector_length_encoding(this);
18818 if (vlen <= 4) {
18819 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18820 } else if (VM_Version::supports_avx2()) {
18821 __ vbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18822 } else {
18823 assert(vlen == 8, "sanity");
18824 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18825 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18826 }
18827 %}
18828 ins_pipe( pipe_slow );
18829 %}
18830
18831 instruct ReplF_reg(vec dst, vlRegF src) %{
18832 predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18833 match(Set dst (Replicate src));
18834 format %{ "replicateF $dst,$src" %}
18835 ins_encode %{
18836 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
18837 %}
18838 ins_pipe( pipe_slow );
18839 %}
18840
18841 instruct ReplF_mem(vec dst, memory mem) %{
18842 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18843 match(Set dst (Replicate (LoadF mem)));
18844 format %{ "replicateF $dst,$mem" %}
18845 ins_encode %{
18846 int vlen_enc = vector_length_encoding(this);
18847 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18848 %}
18849 ins_pipe( pipe_slow );
18850 %}
18851
18852 // Replicate float scalar immediate to be vector by loading from const table.
18853 instruct ReplF_imm(vec dst, immF con) %{
18854 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18855 match(Set dst (Replicate con));
18856 format %{ "replicateF $dst,$con" %}
18857 ins_encode %{
18858 InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant,
18859 VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4));
18860 int vlen = Matcher::vector_length_in_bytes(this);
18861 __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen);
18862 %}
18863 ins_pipe( pipe_slow );
18864 %}
18865
18866 instruct ReplF_zero(vec dst, immF0 zero) %{
18867 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18868 match(Set dst (Replicate zero));
18869 format %{ "replicateF $dst,$zero" %}
18870 ins_encode %{
18871 int vlen_enc = vector_length_encoding(this);
18872 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18873 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18874 } else {
18875 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18876 }
18877 %}
18878 ins_pipe( fpu_reg_reg );
18879 %}
18880
18881 // ====================ReplicateD=======================================
18882
18883 // Replicate double (8 bytes) scalar to be vector
18884 instruct vReplD_reg(vec dst, vlRegD src) %{
18885 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18886 match(Set dst (Replicate src));
18887 format %{ "replicateD $dst,$src" %}
18888 ins_encode %{
18889 uint vlen = Matcher::vector_length(this);
18890 int vlen_enc = vector_length_encoding(this);
18891 if (vlen <= 2) {
18892 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18893 } else if (VM_Version::supports_avx2()) {
18894 __ vbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18895 } else {
18896 assert(vlen == 4, "sanity");
18897 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18898 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18899 }
18900 %}
18901 ins_pipe( pipe_slow );
18902 %}
18903
18904 instruct ReplD_reg(vec dst, vlRegD src) %{
18905 predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18906 match(Set dst (Replicate src));
18907 format %{ "replicateD $dst,$src" %}
18908 ins_encode %{
18909 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
18910 %}
18911 ins_pipe( pipe_slow );
18912 %}
18913
18914 instruct ReplD_mem(vec dst, memory mem) %{
18915 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18916 match(Set dst (Replicate (LoadD mem)));
18917 format %{ "replicateD $dst,$mem" %}
18918 ins_encode %{
18919 if (Matcher::vector_length(this) >= 4) {
18920 int vlen_enc = vector_length_encoding(this);
18921 __ vbroadcastsd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18922 } else {
18923 __ movddup($dst$$XMMRegister, $mem$$Address);
18924 }
18925 %}
18926 ins_pipe( pipe_slow );
18927 %}
18928
18929 // Replicate double (8 byte) scalar immediate to be vector by loading from const table.
18930 instruct ReplD_imm(vec dst, immD con) %{
18931 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18932 match(Set dst (Replicate con));
18933 format %{ "replicateD $dst,$con" %}
18934 ins_encode %{
18935 InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18936 int vlen = Matcher::vector_length_in_bytes(this);
18937 __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen);
18938 %}
18939 ins_pipe( pipe_slow );
18940 %}
18941
18942 instruct ReplD_zero(vec dst, immD0 zero) %{
18943 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18944 match(Set dst (Replicate zero));
18945 format %{ "replicateD $dst,$zero" %}
18946 ins_encode %{
18947 int vlen_enc = vector_length_encoding(this);
18948 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18949 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18950 } else {
18951 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18952 }
18953 %}
18954 ins_pipe( fpu_reg_reg );
18955 %}
18956
18957 // ====================VECTOR INSERT=======================================
18958
18959 instruct insert(vec dst, rRegI val, immU8 idx) %{
18960 predicate(Matcher::vector_length_in_bytes(n) < 32);
18961 match(Set dst (VectorInsert (Binary dst val) idx));
18962 format %{ "vector_insert $dst,$val,$idx" %}
18963 ins_encode %{
18964 assert(UseSSE >= 4, "required");
18965 assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
18966
18967 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18968
18969 assert(is_integral_type(elem_bt), "");
18970 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18971
18972 __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
18973 %}
18974 ins_pipe( pipe_slow );
18975 %}
18976
18977 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
18978 predicate(Matcher::vector_length_in_bytes(n) == 32);
18979 match(Set dst (VectorInsert (Binary src val) idx));
18980 effect(TEMP vtmp);
18981 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18982 ins_encode %{
18983 int vlen_enc = Assembler::AVX_256bit;
18984 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18985 int elem_per_lane = 16/type2aelembytes(elem_bt);
18986 int log2epr = log2(elem_per_lane);
18987
18988 assert(is_integral_type(elem_bt), "sanity");
18989 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18990
18991 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18992 uint y_idx = ($idx$$constant >> log2epr) & 1;
18993 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18994 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18995 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18996 %}
18997 ins_pipe( pipe_slow );
18998 %}
18999
19000 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
19001 predicate(Matcher::vector_length_in_bytes(n) == 64);
19002 match(Set dst (VectorInsert (Binary src val) idx));
19003 effect(TEMP vtmp);
19004 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19005 ins_encode %{
19006 assert(UseAVX > 2, "sanity");
19007
19008 BasicType elem_bt = Matcher::vector_element_basic_type(this);
19009 int elem_per_lane = 16/type2aelembytes(elem_bt);
19010 int log2epr = log2(elem_per_lane);
19011
19012 assert(is_integral_type(elem_bt), "");
19013 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19014
19015 uint x_idx = $idx$$constant & right_n_bits(log2epr);
19016 uint y_idx = ($idx$$constant >> log2epr) & 3;
19017 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19018 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19019 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19020 %}
19021 ins_pipe( pipe_slow );
19022 %}
19023
19024 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
19025 predicate(Matcher::vector_length(n) == 2);
19026 match(Set dst (VectorInsert (Binary dst val) idx));
19027 format %{ "vector_insert $dst,$val,$idx" %}
19028 ins_encode %{
19029 assert(UseSSE >= 4, "required");
19030 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
19031 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19032
19033 __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
19034 %}
19035 ins_pipe( pipe_slow );
19036 %}
19037
19038 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
19039 predicate(Matcher::vector_length(n) == 4);
19040 match(Set dst (VectorInsert (Binary src val) idx));
19041 effect(TEMP vtmp);
19042 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19043 ins_encode %{
19044 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
19045 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19046
19047 uint x_idx = $idx$$constant & right_n_bits(1);
19048 uint y_idx = ($idx$$constant >> 1) & 1;
19049 int vlen_enc = Assembler::AVX_256bit;
19050 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19051 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19052 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19053 %}
19054 ins_pipe( pipe_slow );
19055 %}
19056
19057 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
19058 predicate(Matcher::vector_length(n) == 8);
19059 match(Set dst (VectorInsert (Binary src val) idx));
19060 effect(TEMP vtmp);
19061 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19062 ins_encode %{
19063 assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
19064 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19065
19066 uint x_idx = $idx$$constant & right_n_bits(1);
19067 uint y_idx = ($idx$$constant >> 1) & 3;
19068 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19069 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19070 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19071 %}
19072 ins_pipe( pipe_slow );
19073 %}
19074
19075 instruct insertF(vec dst, regF val, immU8 idx) %{
19076 predicate(Matcher::vector_length(n) < 8);
19077 match(Set dst (VectorInsert (Binary dst val) idx));
19078 format %{ "vector_insert $dst,$val,$idx" %}
19079 ins_encode %{
19080 assert(UseSSE >= 4, "sanity");
19081
19082 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
19083 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19084
19085 uint x_idx = $idx$$constant & right_n_bits(2);
19086 __ insertps($dst$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19087 %}
19088 ins_pipe( pipe_slow );
19089 %}
19090
19091 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
19092 predicate(Matcher::vector_length(n) >= 8);
19093 match(Set dst (VectorInsert (Binary src val) idx));
19094 effect(TEMP vtmp);
19095 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19096 ins_encode %{
19097 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
19098 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19099
19100 int vlen = Matcher::vector_length(this);
19101 uint x_idx = $idx$$constant & right_n_bits(2);
19102 if (vlen == 8) {
19103 uint y_idx = ($idx$$constant >> 2) & 1;
19104 int vlen_enc = Assembler::AVX_256bit;
19105 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19106 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19107 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19108 } else {
19109 assert(vlen == 16, "sanity");
19110 uint y_idx = ($idx$$constant >> 2) & 3;
19111 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19112 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19113 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19114 }
19115 %}
19116 ins_pipe( pipe_slow );
19117 %}
19118
19119 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
19120 predicate(Matcher::vector_length(n) == 2);
19121 match(Set dst (VectorInsert (Binary dst val) idx));
19122 effect(TEMP tmp);
19123 format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
19124 ins_encode %{
19125 assert(UseSSE >= 4, "sanity");
19126 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19127 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19128
19129 __ movq($tmp$$Register, $val$$XMMRegister);
19130 __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
19131 %}
19132 ins_pipe( pipe_slow );
19133 %}
19134
19135 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
19136 predicate(Matcher::vector_length(n) == 4);
19137 match(Set dst (VectorInsert (Binary src val) idx));
19138 effect(TEMP vtmp, TEMP tmp);
19139 format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
19140 ins_encode %{
19141 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19142 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19143
19144 uint x_idx = $idx$$constant & right_n_bits(1);
19145 uint y_idx = ($idx$$constant >> 1) & 1;
19146 int vlen_enc = Assembler::AVX_256bit;
19147 __ movq($tmp$$Register, $val$$XMMRegister);
19148 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19149 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
19150 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19151 %}
19152 ins_pipe( pipe_slow );
19153 %}
19154
19155 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
19156 predicate(Matcher::vector_length(n) == 8);
19157 match(Set dst (VectorInsert (Binary src val) idx));
19158 effect(TEMP tmp, TEMP vtmp);
19159 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19160 ins_encode %{
19161 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19162 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19163
19164 uint x_idx = $idx$$constant & right_n_bits(1);
19165 uint y_idx = ($idx$$constant >> 1) & 3;
19166 __ movq($tmp$$Register, $val$$XMMRegister);
19167 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19168 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
19169 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19170 %}
19171 ins_pipe( pipe_slow );
19172 %}
19173
19174 // ====================REDUCTION ARITHMETIC=======================================
19175
19176 // =======================Int Reduction==========================================
19177
19178 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19179 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
19180 match(Set dst (AddReductionVI src1 src2));
19181 match(Set dst (MulReductionVI src1 src2));
19182 match(Set dst (AndReductionV src1 src2));
19183 match(Set dst ( OrReductionV src1 src2));
19184 match(Set dst (XorReductionV src1 src2));
19185 match(Set dst (MinReductionV src1 src2));
19186 match(Set dst (MaxReductionV src1 src2));
19187 match(Set dst (UMinReductionV src1 src2));
19188 match(Set dst (UMaxReductionV src1 src2));
19189 effect(TEMP vtmp1, TEMP vtmp2);
19190 format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19191 ins_encode %{
19192 int opcode = this->ideal_Opcode();
19193 int vlen = Matcher::vector_length(this, $src2);
19194 __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19195 %}
19196 ins_pipe( pipe_slow );
19197 %}
19198
19199 // =======================Long Reduction==========================================
19200
19201 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19202 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
19203 match(Set dst (AddReductionVL src1 src2));
19204 match(Set dst (MulReductionVL src1 src2));
19205 match(Set dst (AndReductionV src1 src2));
19206 match(Set dst ( OrReductionV src1 src2));
19207 match(Set dst (XorReductionV src1 src2));
19208 match(Set dst (MinReductionV src1 src2));
19209 match(Set dst (MaxReductionV src1 src2));
19210 match(Set dst (UMinReductionV src1 src2));
19211 match(Set dst (UMaxReductionV src1 src2));
19212 effect(TEMP vtmp1, TEMP vtmp2);
19213 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19214 ins_encode %{
19215 int opcode = this->ideal_Opcode();
19216 int vlen = Matcher::vector_length(this, $src2);
19217 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19218 %}
19219 ins_pipe( pipe_slow );
19220 %}
19221
19222 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
19223 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
19224 match(Set dst (AddReductionVL src1 src2));
19225 match(Set dst (MulReductionVL src1 src2));
19226 match(Set dst (AndReductionV src1 src2));
19227 match(Set dst ( OrReductionV src1 src2));
19228 match(Set dst (XorReductionV src1 src2));
19229 match(Set dst (MinReductionV src1 src2));
19230 match(Set dst (MaxReductionV src1 src2));
19231 match(Set dst (UMinReductionV src1 src2));
19232 match(Set dst (UMaxReductionV src1 src2));
19233 effect(TEMP vtmp1, TEMP vtmp2);
19234 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19235 ins_encode %{
19236 int opcode = this->ideal_Opcode();
19237 int vlen = Matcher::vector_length(this, $src2);
19238 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19239 %}
19240 ins_pipe( pipe_slow );
19241 %}
19242
19243 // =======================Float Reduction==========================================
19244
19245 instruct reductionF128(regF dst, vec src, vec vtmp) %{
19246 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) <= 4); // src
19247 match(Set dst (AddReductionVF dst src));
19248 match(Set dst (MulReductionVF dst src));
19249 effect(TEMP dst, TEMP vtmp);
19250 format %{ "vector_reduction_float $dst,$src ; using $vtmp as TEMP" %}
19251 ins_encode %{
19252 int opcode = this->ideal_Opcode();
19253 int vlen = Matcher::vector_length(this, $src);
19254 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19255 %}
19256 ins_pipe( pipe_slow );
19257 %}
19258
19259 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
19260 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19261 match(Set dst (AddReductionVF dst src));
19262 match(Set dst (MulReductionVF dst src));
19263 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19264 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19265 ins_encode %{
19266 int opcode = this->ideal_Opcode();
19267 int vlen = Matcher::vector_length(this, $src);
19268 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19269 %}
19270 ins_pipe( pipe_slow );
19271 %}
19272
19273 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19274 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src
19275 match(Set dst (AddReductionVF dst src));
19276 match(Set dst (MulReductionVF dst src));
19277 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19278 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19279 ins_encode %{
19280 int opcode = this->ideal_Opcode();
19281 int vlen = Matcher::vector_length(this, $src);
19282 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19283 %}
19284 ins_pipe( pipe_slow );
19285 %}
19286
19287
19288 instruct unordered_reduction2F(regF dst, regF src1, vec src2) %{
19289 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19290 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19291 // src1 contains reduction identity
19292 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19293 match(Set dst (AddReductionVF src1 src2));
19294 match(Set dst (MulReductionVF src1 src2));
19295 effect(TEMP dst);
19296 format %{ "vector_reduction_float $dst,$src1,$src2 ;" %}
19297 ins_encode %{
19298 int opcode = this->ideal_Opcode();
19299 int vlen = Matcher::vector_length(this, $src2);
19300 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19301 %}
19302 ins_pipe( pipe_slow );
19303 %}
19304
19305 instruct unordered_reduction4F(regF dst, regF src1, vec src2, vec vtmp) %{
19306 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19307 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19308 // src1 contains reduction identity
19309 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19310 match(Set dst (AddReductionVF src1 src2));
19311 match(Set dst (MulReductionVF src1 src2));
19312 effect(TEMP dst, TEMP vtmp);
19313 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19314 ins_encode %{
19315 int opcode = this->ideal_Opcode();
19316 int vlen = Matcher::vector_length(this, $src2);
19317 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19318 %}
19319 ins_pipe( pipe_slow );
19320 %}
19321
19322 instruct unordered_reduction8F(regF dst, regF src1, vec src2, vec vtmp1, vec vtmp2) %{
19323 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19324 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19325 // src1 contains reduction identity
19326 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19327 match(Set dst (AddReductionVF src1 src2));
19328 match(Set dst (MulReductionVF src1 src2));
19329 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19330 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19331 ins_encode %{
19332 int opcode = this->ideal_Opcode();
19333 int vlen = Matcher::vector_length(this, $src2);
19334 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19335 %}
19336 ins_pipe( pipe_slow );
19337 %}
19338
19339 instruct unordered_reduction16F(regF dst, regF src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19340 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19341 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19342 // src1 contains reduction identity
19343 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src2
19344 match(Set dst (AddReductionVF src1 src2));
19345 match(Set dst (MulReductionVF src1 src2));
19346 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19347 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19348 ins_encode %{
19349 int opcode = this->ideal_Opcode();
19350 int vlen = Matcher::vector_length(this, $src2);
19351 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19352 %}
19353 ins_pipe( pipe_slow );
19354 %}
19355
19356 // =======================Double Reduction==========================================
19357
19358 instruct reduction2D(regD dst, vec src, vec vtmp) %{
19359 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src
19360 match(Set dst (AddReductionVD dst src));
19361 match(Set dst (MulReductionVD dst src));
19362 effect(TEMP dst, TEMP vtmp);
19363 format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
19364 ins_encode %{
19365 int opcode = this->ideal_Opcode();
19366 int vlen = Matcher::vector_length(this, $src);
19367 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19368 %}
19369 ins_pipe( pipe_slow );
19370 %}
19371
19372 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
19373 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src
19374 match(Set dst (AddReductionVD dst src));
19375 match(Set dst (MulReductionVD dst src));
19376 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19377 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19378 ins_encode %{
19379 int opcode = this->ideal_Opcode();
19380 int vlen = Matcher::vector_length(this, $src);
19381 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19382 %}
19383 ins_pipe( pipe_slow );
19384 %}
19385
19386 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19387 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19388 match(Set dst (AddReductionVD dst src));
19389 match(Set dst (MulReductionVD dst src));
19390 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19391 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19392 ins_encode %{
19393 int opcode = this->ideal_Opcode();
19394 int vlen = Matcher::vector_length(this, $src);
19395 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19396 %}
19397 ins_pipe( pipe_slow );
19398 %}
19399
19400 instruct unordered_reduction2D(regD dst, regD src1, vec src2) %{
19401 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19402 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19403 // src1 contains reduction identity
19404 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19405 match(Set dst (AddReductionVD src1 src2));
19406 match(Set dst (MulReductionVD src1 src2));
19407 effect(TEMP dst);
19408 format %{ "vector_reduction_double $dst,$src1,$src2 ;" %}
19409 ins_encode %{
19410 int opcode = this->ideal_Opcode();
19411 int vlen = Matcher::vector_length(this, $src2);
19412 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19413 %}
19414 ins_pipe( pipe_slow );
19415 %}
19416
19417 instruct unordered_reduction4D(regD dst, regD src1, vec src2, vec vtmp) %{
19418 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19419 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19420 // src1 contains reduction identity
19421 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19422 match(Set dst (AddReductionVD src1 src2));
19423 match(Set dst (MulReductionVD src1 src2));
19424 effect(TEMP dst, TEMP vtmp);
19425 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19426 ins_encode %{
19427 int opcode = this->ideal_Opcode();
19428 int vlen = Matcher::vector_length(this, $src2);
19429 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19430 %}
19431 ins_pipe( pipe_slow );
19432 %}
19433
19434 instruct unordered_reduction8D(regD dst, regD src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19435 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19436 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19437 // src1 contains reduction identity
19438 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19439 match(Set dst (AddReductionVD src1 src2));
19440 match(Set dst (MulReductionVD src1 src2));
19441 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19442 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19443 ins_encode %{
19444 int opcode = this->ideal_Opcode();
19445 int vlen = Matcher::vector_length(this, $src2);
19446 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19447 %}
19448 ins_pipe( pipe_slow );
19449 %}
19450
19451 // =======================Byte Reduction==========================================
19452
19453 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19454 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
19455 match(Set dst (AddReductionVI src1 src2));
19456 match(Set dst (AndReductionV src1 src2));
19457 match(Set dst ( OrReductionV src1 src2));
19458 match(Set dst (XorReductionV src1 src2));
19459 match(Set dst (MinReductionV src1 src2));
19460 match(Set dst (MaxReductionV src1 src2));
19461 match(Set dst (UMinReductionV src1 src2));
19462 match(Set dst (UMaxReductionV src1 src2));
19463 effect(TEMP vtmp1, TEMP vtmp2);
19464 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19465 ins_encode %{
19466 int opcode = this->ideal_Opcode();
19467 int vlen = Matcher::vector_length(this, $src2);
19468 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19469 %}
19470 ins_pipe( pipe_slow );
19471 %}
19472
19473 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19474 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
19475 match(Set dst (AddReductionVI src1 src2));
19476 match(Set dst (AndReductionV src1 src2));
19477 match(Set dst ( OrReductionV src1 src2));
19478 match(Set dst (XorReductionV src1 src2));
19479 match(Set dst (MinReductionV src1 src2));
19480 match(Set dst (MaxReductionV src1 src2));
19481 match(Set dst (UMinReductionV src1 src2));
19482 match(Set dst (UMaxReductionV src1 src2));
19483 effect(TEMP vtmp1, TEMP vtmp2);
19484 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19485 ins_encode %{
19486 int opcode = this->ideal_Opcode();
19487 int vlen = Matcher::vector_length(this, $src2);
19488 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19489 %}
19490 ins_pipe( pipe_slow );
19491 %}
19492
19493 // =======================Short Reduction==========================================
19494
19495 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19496 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
19497 match(Set dst (AddReductionVI src1 src2));
19498 match(Set dst (MulReductionVI src1 src2));
19499 match(Set dst (AndReductionV src1 src2));
19500 match(Set dst ( OrReductionV src1 src2));
19501 match(Set dst (XorReductionV src1 src2));
19502 match(Set dst (MinReductionV src1 src2));
19503 match(Set dst (MaxReductionV src1 src2));
19504 match(Set dst (UMinReductionV src1 src2));
19505 match(Set dst (UMaxReductionV src1 src2));
19506 effect(TEMP vtmp1, TEMP vtmp2);
19507 format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19508 ins_encode %{
19509 int opcode = this->ideal_Opcode();
19510 int vlen = Matcher::vector_length(this, $src2);
19511 __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19512 %}
19513 ins_pipe( pipe_slow );
19514 %}
19515
19516 // =======================Mul Reduction==========================================
19517
19518 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19519 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19520 Matcher::vector_length(n->in(2)) <= 32); // src2
19521 match(Set dst (MulReductionVI src1 src2));
19522 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19523 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19524 ins_encode %{
19525 int opcode = this->ideal_Opcode();
19526 int vlen = Matcher::vector_length(this, $src2);
19527 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19528 %}
19529 ins_pipe( pipe_slow );
19530 %}
19531
19532 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19533 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19534 Matcher::vector_length(n->in(2)) == 64); // src2
19535 match(Set dst (MulReductionVI src1 src2));
19536 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19537 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19538 ins_encode %{
19539 int opcode = this->ideal_Opcode();
19540 int vlen = Matcher::vector_length(this, $src2);
19541 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19542 %}
19543 ins_pipe( pipe_slow );
19544 %}
19545
19546 //--------------------Min/Max Float Reduction --------------------
19547 // Float Min Reduction
19548 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19549 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19550 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19551 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19552 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19553 Matcher::vector_length(n->in(2)) == 2);
19554 match(Set dst (MinReductionV src1 src2));
19555 match(Set dst (MaxReductionV src1 src2));
19556 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19557 format %{ "vector_minmax2F_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19558 ins_encode %{
19559 assert(UseAVX > 0, "sanity");
19560
19561 int opcode = this->ideal_Opcode();
19562 int vlen = Matcher::vector_length(this, $src2);
19563 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19564 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19565 %}
19566 ins_pipe( pipe_slow );
19567 %}
19568
19569 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19570 legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19571 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19572 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19573 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19574 Matcher::vector_length(n->in(2)) >= 4);
19575 match(Set dst (MinReductionV src1 src2));
19576 match(Set dst (MaxReductionV src1 src2));
19577 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19578 format %{ "vector_minmaxF_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19579 ins_encode %{
19580 assert(UseAVX > 0, "sanity");
19581
19582 int opcode = this->ideal_Opcode();
19583 int vlen = Matcher::vector_length(this, $src2);
19584 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19585 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19586 %}
19587 ins_pipe( pipe_slow );
19588 %}
19589
19590 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp, legVec atmp,
19591 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19592 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19593 Matcher::vector_length(n->in(2)) == 2);
19594 match(Set dst (MinReductionV dst src));
19595 match(Set dst (MaxReductionV dst src));
19596 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19597 format %{ "vector_minmax2F_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19598 ins_encode %{
19599 assert(UseAVX > 0, "sanity");
19600
19601 int opcode = this->ideal_Opcode();
19602 int vlen = Matcher::vector_length(this, $src);
19603 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19604 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19605 %}
19606 ins_pipe( pipe_slow );
19607 %}
19608
19609
19610 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp, legVec atmp, legVec btmp,
19611 legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19612 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19613 Matcher::vector_length(n->in(2)) >= 4);
19614 match(Set dst (MinReductionV dst src));
19615 match(Set dst (MaxReductionV dst src));
19616 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19617 format %{ "vector_minmaxF_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19618 ins_encode %{
19619 assert(UseAVX > 0, "sanity");
19620
19621 int opcode = this->ideal_Opcode();
19622 int vlen = Matcher::vector_length(this, $src);
19623 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19624 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19625 %}
19626 ins_pipe( pipe_slow );
19627 %}
19628
19629 instruct minmax_reduction2F_avx10_2(regF dst, immF src1, vec src2, vec xtmp1) %{
19630 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19631 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19632 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19633 Matcher::vector_length(n->in(2)) == 2);
19634 match(Set dst (MinReductionV src1 src2));
19635 match(Set dst (MaxReductionV src1 src2));
19636 effect(TEMP dst, TEMP xtmp1);
19637 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 as TEMP" %}
19638 ins_encode %{
19639 int opcode = this->ideal_Opcode();
19640 int vlen = Matcher::vector_length(this, $src2);
19641 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19642 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19643 %}
19644 ins_pipe( pipe_slow );
19645 %}
19646
19647 instruct minmax_reductionF_avx10_2(regF dst, immF src1, vec src2, vec xtmp1, vec xtmp2) %{
19648 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19649 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19650 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19651 Matcher::vector_length(n->in(2)) >= 4);
19652 match(Set dst (MinReductionV src1 src2));
19653 match(Set dst (MaxReductionV src1 src2));
19654 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19655 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 and $xtmp2 as TEMP" %}
19656 ins_encode %{
19657 int opcode = this->ideal_Opcode();
19658 int vlen = Matcher::vector_length(this, $src2);
19659 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19660 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19661 %}
19662 ins_pipe( pipe_slow );
19663 %}
19664
19665 instruct minmax_reduction2F_av_avx10_2(regF dst, vec src, vec xtmp1) %{
19666 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19667 Matcher::vector_length(n->in(2)) == 2);
19668 match(Set dst (MinReductionV dst src));
19669 match(Set dst (MaxReductionV dst src));
19670 effect(TEMP dst, TEMP xtmp1);
19671 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 as TEMP" %}
19672 ins_encode %{
19673 int opcode = this->ideal_Opcode();
19674 int vlen = Matcher::vector_length(this, $src);
19675 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19676 $xtmp1$$XMMRegister);
19677 %}
19678 ins_pipe( pipe_slow );
19679 %}
19680
19681 instruct minmax_reductionF_av_avx10_2(regF dst, vec src, vec xtmp1, vec xtmp2) %{
19682 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19683 Matcher::vector_length(n->in(2)) >= 4);
19684 match(Set dst (MinReductionV dst src));
19685 match(Set dst (MaxReductionV dst src));
19686 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19687 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 and $xtmp2 as TEMP" %}
19688 ins_encode %{
19689 int opcode = this->ideal_Opcode();
19690 int vlen = Matcher::vector_length(this, $src);
19691 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19692 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19693 %}
19694 ins_pipe( pipe_slow );
19695 %}
19696
19697 //--------------------Min Double Reduction --------------------
19698 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19699 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19700 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19701 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19702 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19703 Matcher::vector_length(n->in(2)) == 2);
19704 match(Set dst (MinReductionV src1 src2));
19705 match(Set dst (MaxReductionV src1 src2));
19706 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19707 format %{ "vector_minmax2D_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19708 ins_encode %{
19709 assert(UseAVX > 0, "sanity");
19710
19711 int opcode = this->ideal_Opcode();
19712 int vlen = Matcher::vector_length(this, $src2);
19713 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19714 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19715 %}
19716 ins_pipe( pipe_slow );
19717 %}
19718
19719 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19720 legVec tmp3, legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19721 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19722 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19723 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19724 Matcher::vector_length(n->in(2)) >= 4);
19725 match(Set dst (MinReductionV src1 src2));
19726 match(Set dst (MaxReductionV src1 src2));
19727 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19728 format %{ "vector_minmaxD_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19729 ins_encode %{
19730 assert(UseAVX > 0, "sanity");
19731
19732 int opcode = this->ideal_Opcode();
19733 int vlen = Matcher::vector_length(this, $src2);
19734 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19735 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19736 %}
19737 ins_pipe( pipe_slow );
19738 %}
19739
19740
19741 instruct minmax_reduction2D_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2,
19742 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19743 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19744 Matcher::vector_length(n->in(2)) == 2);
19745 match(Set dst (MinReductionV dst src));
19746 match(Set dst (MaxReductionV dst src));
19747 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19748 format %{ "vector_minmax2D_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19749 ins_encode %{
19750 assert(UseAVX > 0, "sanity");
19751
19752 int opcode = this->ideal_Opcode();
19753 int vlen = Matcher::vector_length(this, $src);
19754 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19755 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19756 %}
19757 ins_pipe( pipe_slow );
19758 %}
19759
19760 instruct minmax_reductionD_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2, legVec tmp3,
19761 legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19762 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19763 Matcher::vector_length(n->in(2)) >= 4);
19764 match(Set dst (MinReductionV dst src));
19765 match(Set dst (MaxReductionV dst src));
19766 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19767 format %{ "vector_minmaxD_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19768 ins_encode %{
19769 assert(UseAVX > 0, "sanity");
19770
19771 int opcode = this->ideal_Opcode();
19772 int vlen = Matcher::vector_length(this, $src);
19773 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19774 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19775 %}
19776 ins_pipe( pipe_slow );
19777 %}
19778
19779 instruct minmax_reduction2D_avx10_2(regD dst, immD src1, vec src2, vec xtmp1) %{
19780 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19781 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19782 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19783 Matcher::vector_length(n->in(2)) == 2);
19784 match(Set dst (MinReductionV src1 src2));
19785 match(Set dst (MaxReductionV src1 src2));
19786 effect(TEMP dst, TEMP xtmp1);
19787 format %{ "vector_minmax2D_reduction $dst, $src1, $src2 ; using $xtmp1 as TEMP" %}
19788 ins_encode %{
19789 int opcode = this->ideal_Opcode();
19790 int vlen = Matcher::vector_length(this, $src2);
19791 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg,
19792 xnoreg, xnoreg, $xtmp1$$XMMRegister);
19793 %}
19794 ins_pipe( pipe_slow );
19795 %}
19796
19797 instruct minmax_reductionD_avx10_2(regD dst, immD src1, vec src2, vec xtmp1, vec xtmp2) %{
19798 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19799 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19800 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19801 Matcher::vector_length(n->in(2)) >= 4);
19802 match(Set dst (MinReductionV src1 src2));
19803 match(Set dst (MaxReductionV src1 src2));
19804 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19805 format %{ "vector_minmaxD_reduction $dst, $src1, $src2 ; using $xtmp1 and $xtmp2 as TEMP" %}
19806 ins_encode %{
19807 int opcode = this->ideal_Opcode();
19808 int vlen = Matcher::vector_length(this, $src2);
19809 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19810 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19811 %}
19812 ins_pipe( pipe_slow );
19813 %}
19814
19815
19816 instruct minmax_reduction2D_av_avx10_2(regD dst, vec src, vec xtmp1) %{
19817 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19818 Matcher::vector_length(n->in(2)) == 2);
19819 match(Set dst (MinReductionV dst src));
19820 match(Set dst (MaxReductionV dst src));
19821 effect(TEMP dst, TEMP xtmp1);
19822 format %{ "vector_minmax2D_reduction $dst, $src ; using $xtmp1 as TEMP" %}
19823 ins_encode %{
19824 int opcode = this->ideal_Opcode();
19825 int vlen = Matcher::vector_length(this, $src);
19826 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19827 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19828 %}
19829 ins_pipe( pipe_slow );
19830 %}
19831
19832 instruct minmax_reductionD_av_avx10_2(regD dst, vec src, vec xtmp1, vec xtmp2) %{
19833 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19834 Matcher::vector_length(n->in(2)) >= 4);
19835 match(Set dst (MinReductionV dst src));
19836 match(Set dst (MaxReductionV dst src));
19837 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19838 format %{ "vector_minmaxD_reduction $dst, $src ; using $xtmp1 and $xtmp2 as TEMP" %}
19839 ins_encode %{
19840 int opcode = this->ideal_Opcode();
19841 int vlen = Matcher::vector_length(this, $src);
19842 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19843 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19844 %}
19845 ins_pipe( pipe_slow );
19846 %}
19847
19848 // ====================VECTOR ARITHMETIC=======================================
19849
19850 // --------------------------------- ADD --------------------------------------
19851
19852 // Bytes vector add
19853 instruct vaddB(vec dst, vec src) %{
19854 predicate(UseAVX == 0);
19855 match(Set dst (AddVB dst src));
19856 format %{ "paddb $dst,$src\t! add packedB" %}
19857 ins_encode %{
19858 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
19859 %}
19860 ins_pipe( pipe_slow );
19861 %}
19862
19863 instruct vaddB_reg(vec dst, vec src1, vec src2) %{
19864 predicate(UseAVX > 0);
19865 match(Set dst (AddVB src1 src2));
19866 format %{ "vpaddb $dst,$src1,$src2\t! add packedB" %}
19867 ins_encode %{
19868 int vlen_enc = vector_length_encoding(this);
19869 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19870 %}
19871 ins_pipe( pipe_slow );
19872 %}
19873
19874 instruct vaddB_mem(vec dst, vec src, memory mem) %{
19875 predicate((UseAVX > 0) &&
19876 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19877 match(Set dst (AddVB src (LoadVector mem)));
19878 format %{ "vpaddb $dst,$src,$mem\t! add packedB" %}
19879 ins_encode %{
19880 int vlen_enc = vector_length_encoding(this);
19881 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19882 %}
19883 ins_pipe( pipe_slow );
19884 %}
19885
19886 // Shorts/Chars vector add
19887 instruct vaddS(vec dst, vec src) %{
19888 predicate(UseAVX == 0);
19889 match(Set dst (AddVS dst src));
19890 format %{ "paddw $dst,$src\t! add packedS" %}
19891 ins_encode %{
19892 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
19893 %}
19894 ins_pipe( pipe_slow );
19895 %}
19896
19897 instruct vaddS_reg(vec dst, vec src1, vec src2) %{
19898 predicate(UseAVX > 0);
19899 match(Set dst (AddVS src1 src2));
19900 format %{ "vpaddw $dst,$src1,$src2\t! add packedS" %}
19901 ins_encode %{
19902 int vlen_enc = vector_length_encoding(this);
19903 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19904 %}
19905 ins_pipe( pipe_slow );
19906 %}
19907
19908 instruct vaddS_mem(vec dst, vec src, memory mem) %{
19909 predicate((UseAVX > 0) &&
19910 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19911 match(Set dst (AddVS src (LoadVector mem)));
19912 format %{ "vpaddw $dst,$src,$mem\t! add packedS" %}
19913 ins_encode %{
19914 int vlen_enc = vector_length_encoding(this);
19915 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19916 %}
19917 ins_pipe( pipe_slow );
19918 %}
19919
19920 // Integers vector add
19921 instruct vaddI(vec dst, vec src) %{
19922 predicate(UseAVX == 0);
19923 match(Set dst (AddVI dst src));
19924 format %{ "paddd $dst,$src\t! add packedI" %}
19925 ins_encode %{
19926 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
19927 %}
19928 ins_pipe( pipe_slow );
19929 %}
19930
19931 instruct vaddI_reg(vec dst, vec src1, vec src2) %{
19932 predicate(UseAVX > 0);
19933 match(Set dst (AddVI src1 src2));
19934 format %{ "vpaddd $dst,$src1,$src2\t! add packedI" %}
19935 ins_encode %{
19936 int vlen_enc = vector_length_encoding(this);
19937 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19938 %}
19939 ins_pipe( pipe_slow );
19940 %}
19941
19942
19943 instruct vaddI_mem(vec dst, vec src, memory mem) %{
19944 predicate((UseAVX > 0) &&
19945 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19946 match(Set dst (AddVI src (LoadVector mem)));
19947 format %{ "vpaddd $dst,$src,$mem\t! add packedI" %}
19948 ins_encode %{
19949 int vlen_enc = vector_length_encoding(this);
19950 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19951 %}
19952 ins_pipe( pipe_slow );
19953 %}
19954
19955 // Longs vector add
19956 instruct vaddL(vec dst, vec src) %{
19957 predicate(UseAVX == 0);
19958 match(Set dst (AddVL dst src));
19959 format %{ "paddq $dst,$src\t! add packedL" %}
19960 ins_encode %{
19961 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
19962 %}
19963 ins_pipe( pipe_slow );
19964 %}
19965
19966 instruct vaddL_reg(vec dst, vec src1, vec src2) %{
19967 predicate(UseAVX > 0);
19968 match(Set dst (AddVL src1 src2));
19969 format %{ "vpaddq $dst,$src1,$src2\t! add packedL" %}
19970 ins_encode %{
19971 int vlen_enc = vector_length_encoding(this);
19972 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19973 %}
19974 ins_pipe( pipe_slow );
19975 %}
19976
19977 instruct vaddL_mem(vec dst, vec src, memory mem) %{
19978 predicate((UseAVX > 0) &&
19979 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19980 match(Set dst (AddVL src (LoadVector mem)));
19981 format %{ "vpaddq $dst,$src,$mem\t! add packedL" %}
19982 ins_encode %{
19983 int vlen_enc = vector_length_encoding(this);
19984 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19985 %}
19986 ins_pipe( pipe_slow );
19987 %}
19988
19989 // Floats vector add
19990 instruct vaddF(vec dst, vec src) %{
19991 predicate(UseAVX == 0);
19992 match(Set dst (AddVF dst src));
19993 format %{ "addps $dst,$src\t! add packedF" %}
19994 ins_encode %{
19995 __ addps($dst$$XMMRegister, $src$$XMMRegister);
19996 %}
19997 ins_pipe( pipe_slow );
19998 %}
19999
20000 instruct vaddF_reg(vec dst, vec src1, vec src2) %{
20001 predicate(UseAVX > 0);
20002 match(Set dst (AddVF src1 src2));
20003 format %{ "vaddps $dst,$src1,$src2\t! add packedF" %}
20004 ins_encode %{
20005 int vlen_enc = vector_length_encoding(this);
20006 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20007 %}
20008 ins_pipe( pipe_slow );
20009 %}
20010
20011 instruct vaddF_mem(vec dst, vec src, memory mem) %{
20012 predicate((UseAVX > 0) &&
20013 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20014 match(Set dst (AddVF src (LoadVector mem)));
20015 format %{ "vaddps $dst,$src,$mem\t! add packedF" %}
20016 ins_encode %{
20017 int vlen_enc = vector_length_encoding(this);
20018 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20019 %}
20020 ins_pipe( pipe_slow );
20021 %}
20022
20023 // Doubles vector add
20024 instruct vaddD(vec dst, vec src) %{
20025 predicate(UseAVX == 0);
20026 match(Set dst (AddVD dst src));
20027 format %{ "addpd $dst,$src\t! add packedD" %}
20028 ins_encode %{
20029 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
20030 %}
20031 ins_pipe( pipe_slow );
20032 %}
20033
20034 instruct vaddD_reg(vec dst, vec src1, vec src2) %{
20035 predicate(UseAVX > 0);
20036 match(Set dst (AddVD src1 src2));
20037 format %{ "vaddpd $dst,$src1,$src2\t! add packedD" %}
20038 ins_encode %{
20039 int vlen_enc = vector_length_encoding(this);
20040 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20041 %}
20042 ins_pipe( pipe_slow );
20043 %}
20044
20045 instruct vaddD_mem(vec dst, vec src, memory mem) %{
20046 predicate((UseAVX > 0) &&
20047 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20048 match(Set dst (AddVD src (LoadVector mem)));
20049 format %{ "vaddpd $dst,$src,$mem\t! add packedD" %}
20050 ins_encode %{
20051 int vlen_enc = vector_length_encoding(this);
20052 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20053 %}
20054 ins_pipe( pipe_slow );
20055 %}
20056
20057 // --------------------------------- SUB --------------------------------------
20058
20059 // Bytes vector sub
20060 instruct vsubB(vec dst, vec src) %{
20061 predicate(UseAVX == 0);
20062 match(Set dst (SubVB dst src));
20063 format %{ "psubb $dst,$src\t! sub packedB" %}
20064 ins_encode %{
20065 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
20066 %}
20067 ins_pipe( pipe_slow );
20068 %}
20069
20070 instruct vsubB_reg(vec dst, vec src1, vec src2) %{
20071 predicate(UseAVX > 0);
20072 match(Set dst (SubVB src1 src2));
20073 format %{ "vpsubb $dst,$src1,$src2\t! sub packedB" %}
20074 ins_encode %{
20075 int vlen_enc = vector_length_encoding(this);
20076 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20077 %}
20078 ins_pipe( pipe_slow );
20079 %}
20080
20081 instruct vsubB_mem(vec dst, vec src, memory mem) %{
20082 predicate((UseAVX > 0) &&
20083 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20084 match(Set dst (SubVB src (LoadVector mem)));
20085 format %{ "vpsubb $dst,$src,$mem\t! sub packedB" %}
20086 ins_encode %{
20087 int vlen_enc = vector_length_encoding(this);
20088 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20089 %}
20090 ins_pipe( pipe_slow );
20091 %}
20092
20093 // Shorts/Chars vector sub
20094 instruct vsubS(vec dst, vec src) %{
20095 predicate(UseAVX == 0);
20096 match(Set dst (SubVS dst src));
20097 format %{ "psubw $dst,$src\t! sub packedS" %}
20098 ins_encode %{
20099 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
20100 %}
20101 ins_pipe( pipe_slow );
20102 %}
20103
20104
20105 instruct vsubS_reg(vec dst, vec src1, vec src2) %{
20106 predicate(UseAVX > 0);
20107 match(Set dst (SubVS src1 src2));
20108 format %{ "vpsubw $dst,$src1,$src2\t! sub packedS" %}
20109 ins_encode %{
20110 int vlen_enc = vector_length_encoding(this);
20111 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20112 %}
20113 ins_pipe( pipe_slow );
20114 %}
20115
20116 instruct vsubS_mem(vec dst, vec src, memory mem) %{
20117 predicate((UseAVX > 0) &&
20118 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20119 match(Set dst (SubVS src (LoadVector mem)));
20120 format %{ "vpsubw $dst,$src,$mem\t! sub packedS" %}
20121 ins_encode %{
20122 int vlen_enc = vector_length_encoding(this);
20123 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20124 %}
20125 ins_pipe( pipe_slow );
20126 %}
20127
20128 // Integers vector sub
20129 instruct vsubI(vec dst, vec src) %{
20130 predicate(UseAVX == 0);
20131 match(Set dst (SubVI dst src));
20132 format %{ "psubd $dst,$src\t! sub packedI" %}
20133 ins_encode %{
20134 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
20135 %}
20136 ins_pipe( pipe_slow );
20137 %}
20138
20139 instruct vsubI_reg(vec dst, vec src1, vec src2) %{
20140 predicate(UseAVX > 0);
20141 match(Set dst (SubVI src1 src2));
20142 format %{ "vpsubd $dst,$src1,$src2\t! sub packedI" %}
20143 ins_encode %{
20144 int vlen_enc = vector_length_encoding(this);
20145 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20146 %}
20147 ins_pipe( pipe_slow );
20148 %}
20149
20150 instruct vsubI_mem(vec dst, vec src, memory mem) %{
20151 predicate((UseAVX > 0) &&
20152 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20153 match(Set dst (SubVI src (LoadVector mem)));
20154 format %{ "vpsubd $dst,$src,$mem\t! sub packedI" %}
20155 ins_encode %{
20156 int vlen_enc = vector_length_encoding(this);
20157 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20158 %}
20159 ins_pipe( pipe_slow );
20160 %}
20161
20162 // Longs vector sub
20163 instruct vsubL(vec dst, vec src) %{
20164 predicate(UseAVX == 0);
20165 match(Set dst (SubVL dst src));
20166 format %{ "psubq $dst,$src\t! sub packedL" %}
20167 ins_encode %{
20168 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
20169 %}
20170 ins_pipe( pipe_slow );
20171 %}
20172
20173 instruct vsubL_reg(vec dst, vec src1, vec src2) %{
20174 predicate(UseAVX > 0);
20175 match(Set dst (SubVL src1 src2));
20176 format %{ "vpsubq $dst,$src1,$src2\t! sub packedL" %}
20177 ins_encode %{
20178 int vlen_enc = vector_length_encoding(this);
20179 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20180 %}
20181 ins_pipe( pipe_slow );
20182 %}
20183
20184
20185 instruct vsubL_mem(vec dst, vec src, memory mem) %{
20186 predicate((UseAVX > 0) &&
20187 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20188 match(Set dst (SubVL src (LoadVector mem)));
20189 format %{ "vpsubq $dst,$src,$mem\t! sub packedL" %}
20190 ins_encode %{
20191 int vlen_enc = vector_length_encoding(this);
20192 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20193 %}
20194 ins_pipe( pipe_slow );
20195 %}
20196
20197 // Floats vector sub
20198 instruct vsubF(vec dst, vec src) %{
20199 predicate(UseAVX == 0);
20200 match(Set dst (SubVF dst src));
20201 format %{ "subps $dst,$src\t! sub packedF" %}
20202 ins_encode %{
20203 __ subps($dst$$XMMRegister, $src$$XMMRegister);
20204 %}
20205 ins_pipe( pipe_slow );
20206 %}
20207
20208 instruct vsubF_reg(vec dst, vec src1, vec src2) %{
20209 predicate(UseAVX > 0);
20210 match(Set dst (SubVF src1 src2));
20211 format %{ "vsubps $dst,$src1,$src2\t! sub packedF" %}
20212 ins_encode %{
20213 int vlen_enc = vector_length_encoding(this);
20214 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20215 %}
20216 ins_pipe( pipe_slow );
20217 %}
20218
20219 instruct vsubF_mem(vec dst, vec src, memory mem) %{
20220 predicate((UseAVX > 0) &&
20221 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20222 match(Set dst (SubVF src (LoadVector mem)));
20223 format %{ "vsubps $dst,$src,$mem\t! sub packedF" %}
20224 ins_encode %{
20225 int vlen_enc = vector_length_encoding(this);
20226 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20227 %}
20228 ins_pipe( pipe_slow );
20229 %}
20230
20231 // Doubles vector sub
20232 instruct vsubD(vec dst, vec src) %{
20233 predicate(UseAVX == 0);
20234 match(Set dst (SubVD dst src));
20235 format %{ "subpd $dst,$src\t! sub packedD" %}
20236 ins_encode %{
20237 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
20238 %}
20239 ins_pipe( pipe_slow );
20240 %}
20241
20242 instruct vsubD_reg(vec dst, vec src1, vec src2) %{
20243 predicate(UseAVX > 0);
20244 match(Set dst (SubVD src1 src2));
20245 format %{ "vsubpd $dst,$src1,$src2\t! sub packedD" %}
20246 ins_encode %{
20247 int vlen_enc = vector_length_encoding(this);
20248 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20249 %}
20250 ins_pipe( pipe_slow );
20251 %}
20252
20253 instruct vsubD_mem(vec dst, vec src, memory mem) %{
20254 predicate((UseAVX > 0) &&
20255 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20256 match(Set dst (SubVD src (LoadVector mem)));
20257 format %{ "vsubpd $dst,$src,$mem\t! sub packedD" %}
20258 ins_encode %{
20259 int vlen_enc = vector_length_encoding(this);
20260 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20261 %}
20262 ins_pipe( pipe_slow );
20263 %}
20264
20265 // --------------------------------- MUL --------------------------------------
20266
20267 // Byte vector mul
20268 instruct vmul8B(vec dst, vec src1, vec src2, vec xtmp) %{
20269 predicate(Matcher::vector_length_in_bytes(n) <= 8);
20270 match(Set dst (MulVB src1 src2));
20271 effect(TEMP dst, TEMP xtmp);
20272 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20273 ins_encode %{
20274 assert(UseSSE > 3, "required");
20275 __ pmovsxbw($dst$$XMMRegister, $src1$$XMMRegister);
20276 __ pmovsxbw($xtmp$$XMMRegister, $src2$$XMMRegister);
20277 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
20278 __ psllw($dst$$XMMRegister, 8);
20279 __ psrlw($dst$$XMMRegister, 8);
20280 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
20281 %}
20282 ins_pipe( pipe_slow );
20283 %}
20284
20285 instruct vmulB(vec dst, vec src1, vec src2, vec xtmp) %{
20286 predicate(UseAVX == 0 && Matcher::vector_length_in_bytes(n) > 8);
20287 match(Set dst (MulVB src1 src2));
20288 effect(TEMP dst, TEMP xtmp);
20289 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20290 ins_encode %{
20291 assert(UseSSE > 3, "required");
20292 // Odd-index elements
20293 __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
20294 __ psrlw($dst$$XMMRegister, 8);
20295 __ movdqu($xtmp$$XMMRegister, $src2$$XMMRegister);
20296 __ psrlw($xtmp$$XMMRegister, 8);
20297 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
20298 __ psllw($dst$$XMMRegister, 8);
20299 // Even-index elements
20300 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20301 __ pmullw($xtmp$$XMMRegister, $src2$$XMMRegister);
20302 __ psllw($xtmp$$XMMRegister, 8);
20303 __ psrlw($xtmp$$XMMRegister, 8);
20304 // Combine
20305 __ por($dst$$XMMRegister, $xtmp$$XMMRegister);
20306 %}
20307 ins_pipe( pipe_slow );
20308 %}
20309
20310 instruct vmulB_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20311 predicate(UseAVX > 0 && Matcher::vector_length_in_bytes(n) > 8);
20312 match(Set dst (MulVB src1 src2));
20313 effect(TEMP xtmp1, TEMP xtmp2);
20314 format %{ "vmulVB $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20315 ins_encode %{
20316 int vlen_enc = vector_length_encoding(this);
20317 // Odd-index elements
20318 __ vpsrlw($xtmp2$$XMMRegister, $src1$$XMMRegister, 8, vlen_enc);
20319 __ vpsrlw($xtmp1$$XMMRegister, $src2$$XMMRegister, 8, vlen_enc);
20320 __ vpmullw($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20321 __ vpsllw($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 8, vlen_enc);
20322 // Even-index elements
20323 __ vpmullw($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20324 __ vpsllw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20325 __ vpsrlw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20326 // Combine
20327 __ vpor($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20328 %}
20329 ins_pipe( pipe_slow );
20330 %}
20331
20332 // Shorts/Chars vector mul
20333 instruct vmulS(vec dst, vec src) %{
20334 predicate(UseAVX == 0);
20335 match(Set dst (MulVS dst src));
20336 format %{ "pmullw $dst,$src\t! mul packedS" %}
20337 ins_encode %{
20338 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
20339 %}
20340 ins_pipe( pipe_slow );
20341 %}
20342
20343 instruct vmulS_reg(vec dst, vec src1, vec src2) %{
20344 predicate(UseAVX > 0);
20345 match(Set dst (MulVS src1 src2));
20346 format %{ "vpmullw $dst,$src1,$src2\t! mul packedS" %}
20347 ins_encode %{
20348 int vlen_enc = vector_length_encoding(this);
20349 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20350 %}
20351 ins_pipe( pipe_slow );
20352 %}
20353
20354 instruct vmulS_mem(vec dst, vec src, memory mem) %{
20355 predicate((UseAVX > 0) &&
20356 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20357 match(Set dst (MulVS src (LoadVector mem)));
20358 format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
20359 ins_encode %{
20360 int vlen_enc = vector_length_encoding(this);
20361 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20362 %}
20363 ins_pipe( pipe_slow );
20364 %}
20365
20366 // Integers vector mul
20367 instruct vmulI(vec dst, vec src) %{
20368 predicate(UseAVX == 0);
20369 match(Set dst (MulVI dst src));
20370 format %{ "pmulld $dst,$src\t! mul packedI" %}
20371 ins_encode %{
20372 assert(UseSSE > 3, "required");
20373 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
20374 %}
20375 ins_pipe( pipe_slow );
20376 %}
20377
20378 instruct vmulI_reg(vec dst, vec src1, vec src2) %{
20379 predicate(UseAVX > 0);
20380 match(Set dst (MulVI src1 src2));
20381 format %{ "vpmulld $dst,$src1,$src2\t! mul packedI" %}
20382 ins_encode %{
20383 int vlen_enc = vector_length_encoding(this);
20384 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20385 %}
20386 ins_pipe( pipe_slow );
20387 %}
20388
20389 instruct vmulI_mem(vec dst, vec src, memory mem) %{
20390 predicate((UseAVX > 0) &&
20391 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20392 match(Set dst (MulVI src (LoadVector mem)));
20393 format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
20394 ins_encode %{
20395 int vlen_enc = vector_length_encoding(this);
20396 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20397 %}
20398 ins_pipe( pipe_slow );
20399 %}
20400
20401 // Longs vector mul
20402 instruct evmulL_reg(vec dst, vec src1, vec src2) %{
20403 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20404 VM_Version::supports_avx512dq()) ||
20405 VM_Version::supports_avx512vldq());
20406 match(Set dst (MulVL src1 src2));
20407 ins_cost(500);
20408 format %{ "evpmullq $dst,$src1,$src2\t! mul packedL" %}
20409 ins_encode %{
20410 assert(UseAVX > 2, "required");
20411 int vlen_enc = vector_length_encoding(this);
20412 __ evpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20413 %}
20414 ins_pipe( pipe_slow );
20415 %}
20416
20417 instruct evmulL_mem(vec dst, vec src, memory mem) %{
20418 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20419 VM_Version::supports_avx512dq()) ||
20420 (Matcher::vector_length_in_bytes(n) > 8 &&
20421 VM_Version::supports_avx512vldq()));
20422 match(Set dst (MulVL src (LoadVector mem)));
20423 format %{ "evpmullq $dst,$src,$mem\t! mul packedL" %}
20424 ins_cost(500);
20425 ins_encode %{
20426 assert(UseAVX > 2, "required");
20427 int vlen_enc = vector_length_encoding(this);
20428 __ evpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20429 %}
20430 ins_pipe( pipe_slow );
20431 %}
20432
20433 instruct vmulL(vec dst, vec src1, vec src2, vec xtmp) %{
20434 predicate(UseAVX == 0);
20435 match(Set dst (MulVL src1 src2));
20436 ins_cost(500);
20437 effect(TEMP dst, TEMP xtmp);
20438 format %{ "mulVL $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20439 ins_encode %{
20440 assert(VM_Version::supports_sse4_1(), "required");
20441 // Get the lo-hi products, only the lower 32 bits is in concerns
20442 __ pshufd($xtmp$$XMMRegister, $src2$$XMMRegister, 0xB1);
20443 __ pmulld($xtmp$$XMMRegister, $src1$$XMMRegister);
20444 __ pshufd($dst$$XMMRegister, $xtmp$$XMMRegister, 0xB1);
20445 __ paddd($dst$$XMMRegister, $xtmp$$XMMRegister);
20446 __ psllq($dst$$XMMRegister, 32);
20447 // Get the lo-lo products
20448 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20449 __ pmuludq($xtmp$$XMMRegister, $src2$$XMMRegister);
20450 __ paddq($dst$$XMMRegister, $xtmp$$XMMRegister);
20451 %}
20452 ins_pipe( pipe_slow );
20453 %}
20454
20455 instruct vmulL_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20456 predicate(UseAVX > 0 &&
20457 ((Matcher::vector_length_in_bytes(n) == 64 &&
20458 !VM_Version::supports_avx512dq()) ||
20459 (Matcher::vector_length_in_bytes(n) < 64 &&
20460 !VM_Version::supports_avx512vldq())));
20461 match(Set dst (MulVL src1 src2));
20462 effect(TEMP xtmp1, TEMP xtmp2);
20463 ins_cost(500);
20464 format %{ "vmulVL $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20465 ins_encode %{
20466 int vlen_enc = vector_length_encoding(this);
20467 // Get the lo-hi products, only the lower 32 bits is in concerns
20468 __ vpshufd($xtmp1$$XMMRegister, $src2$$XMMRegister, 0xB1, vlen_enc);
20469 __ vpmulld($xtmp1$$XMMRegister, $src1$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20470 __ vpshufd($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, 0xB1, vlen_enc);
20471 __ vpaddd($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20472 __ vpsllq($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 32, vlen_enc);
20473 // Get the lo-lo products
20474 __ vpmuludq($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20475 __ vpaddq($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20476 %}
20477 ins_pipe( pipe_slow );
20478 %}
20479
20480 instruct vmuludq_reg(vec dst, vec src1, vec src2) %{
20481 predicate(UseAVX > 0 && n->as_MulVL()->has_uint_inputs());
20482 match(Set dst (MulVL src1 src2));
20483 ins_cost(100);
20484 format %{ "vpmuludq $dst,$src1,$src2\t! muludq packedL" %}
20485 ins_encode %{
20486 int vlen_enc = vector_length_encoding(this);
20487 __ vpmuludq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20488 %}
20489 ins_pipe( pipe_slow );
20490 %}
20491
20492 instruct vmuldq_reg(vec dst, vec src1, vec src2) %{
20493 predicate(UseAVX > 0 && n->as_MulVL()->has_int_inputs());
20494 match(Set dst (MulVL src1 src2));
20495 ins_cost(100);
20496 format %{ "vpmuldq $dst,$src1,$src2\t! muldq packedL" %}
20497 ins_encode %{
20498 int vlen_enc = vector_length_encoding(this);
20499 __ vpmuldq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20500 %}
20501 ins_pipe( pipe_slow );
20502 %}
20503
20504 // Floats vector mul
20505 instruct vmulF(vec dst, vec src) %{
20506 predicate(UseAVX == 0);
20507 match(Set dst (MulVF dst src));
20508 format %{ "mulps $dst,$src\t! mul packedF" %}
20509 ins_encode %{
20510 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
20511 %}
20512 ins_pipe( pipe_slow );
20513 %}
20514
20515 instruct vmulF_reg(vec dst, vec src1, vec src2) %{
20516 predicate(UseAVX > 0);
20517 match(Set dst (MulVF src1 src2));
20518 format %{ "vmulps $dst,$src1,$src2\t! mul packedF" %}
20519 ins_encode %{
20520 int vlen_enc = vector_length_encoding(this);
20521 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20522 %}
20523 ins_pipe( pipe_slow );
20524 %}
20525
20526 instruct vmulF_mem(vec dst, vec src, memory mem) %{
20527 predicate((UseAVX > 0) &&
20528 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20529 match(Set dst (MulVF src (LoadVector mem)));
20530 format %{ "vmulps $dst,$src,$mem\t! mul packedF" %}
20531 ins_encode %{
20532 int vlen_enc = vector_length_encoding(this);
20533 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20534 %}
20535 ins_pipe( pipe_slow );
20536 %}
20537
20538 // Doubles vector mul
20539 instruct vmulD(vec dst, vec src) %{
20540 predicate(UseAVX == 0);
20541 match(Set dst (MulVD dst src));
20542 format %{ "mulpd $dst,$src\t! mul packedD" %}
20543 ins_encode %{
20544 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
20545 %}
20546 ins_pipe( pipe_slow );
20547 %}
20548
20549 instruct vmulD_reg(vec dst, vec src1, vec src2) %{
20550 predicate(UseAVX > 0);
20551 match(Set dst (MulVD src1 src2));
20552 format %{ "vmulpd $dst,$src1,$src2\t! mul packedD" %}
20553 ins_encode %{
20554 int vlen_enc = vector_length_encoding(this);
20555 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20556 %}
20557 ins_pipe( pipe_slow );
20558 %}
20559
20560 instruct vmulD_mem(vec dst, vec src, memory mem) %{
20561 predicate((UseAVX > 0) &&
20562 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20563 match(Set dst (MulVD src (LoadVector mem)));
20564 format %{ "vmulpd $dst,$src,$mem\t! mul packedD" %}
20565 ins_encode %{
20566 int vlen_enc = vector_length_encoding(this);
20567 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20568 %}
20569 ins_pipe( pipe_slow );
20570 %}
20571
20572 // --------------------------------- DIV --------------------------------------
20573
20574 // Floats vector div
20575 instruct vdivF(vec dst, vec src) %{
20576 predicate(UseAVX == 0);
20577 match(Set dst (DivVF dst src));
20578 format %{ "divps $dst,$src\t! div packedF" %}
20579 ins_encode %{
20580 __ divps($dst$$XMMRegister, $src$$XMMRegister);
20581 %}
20582 ins_pipe( pipe_slow );
20583 %}
20584
20585 instruct vdivF_reg(vec dst, vec src1, vec src2) %{
20586 predicate(UseAVX > 0);
20587 match(Set dst (DivVF src1 src2));
20588 format %{ "vdivps $dst,$src1,$src2\t! div packedF" %}
20589 ins_encode %{
20590 int vlen_enc = vector_length_encoding(this);
20591 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20592 %}
20593 ins_pipe( pipe_slow );
20594 %}
20595
20596 instruct vdivF_mem(vec dst, vec src, memory mem) %{
20597 predicate((UseAVX > 0) &&
20598 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20599 match(Set dst (DivVF src (LoadVector mem)));
20600 format %{ "vdivps $dst,$src,$mem\t! div packedF" %}
20601 ins_encode %{
20602 int vlen_enc = vector_length_encoding(this);
20603 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20604 %}
20605 ins_pipe( pipe_slow );
20606 %}
20607
20608 // Doubles vector div
20609 instruct vdivD(vec dst, vec src) %{
20610 predicate(UseAVX == 0);
20611 match(Set dst (DivVD dst src));
20612 format %{ "divpd $dst,$src\t! div packedD" %}
20613 ins_encode %{
20614 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
20615 %}
20616 ins_pipe( pipe_slow );
20617 %}
20618
20619 instruct vdivD_reg(vec dst, vec src1, vec src2) %{
20620 predicate(UseAVX > 0);
20621 match(Set dst (DivVD src1 src2));
20622 format %{ "vdivpd $dst,$src1,$src2\t! div packedD" %}
20623 ins_encode %{
20624 int vlen_enc = vector_length_encoding(this);
20625 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20626 %}
20627 ins_pipe( pipe_slow );
20628 %}
20629
20630 instruct vdivD_mem(vec dst, vec src, memory mem) %{
20631 predicate((UseAVX > 0) &&
20632 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20633 match(Set dst (DivVD src (LoadVector mem)));
20634 format %{ "vdivpd $dst,$src,$mem\t! div packedD" %}
20635 ins_encode %{
20636 int vlen_enc = vector_length_encoding(this);
20637 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20638 %}
20639 ins_pipe( pipe_slow );
20640 %}
20641
20642 // ------------------------------ MinMax ---------------------------------------
20643
20644 // Byte, Short, Int vector Min/Max
20645 instruct minmax_reg_sse(vec dst, vec src) %{
20646 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20647 UseAVX == 0);
20648 match(Set dst (MinV dst src));
20649 match(Set dst (MaxV dst src));
20650 format %{ "vector_minmax $dst,$src\t! " %}
20651 ins_encode %{
20652 assert(UseSSE >= 4, "required");
20653
20654 int opcode = this->ideal_Opcode();
20655 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20656 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
20657 %}
20658 ins_pipe( pipe_slow );
20659 %}
20660
20661 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
20662 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20663 UseAVX > 0);
20664 match(Set dst (MinV src1 src2));
20665 match(Set dst (MaxV src1 src2));
20666 format %{ "vector_minmax $dst,$src1,$src2\t! " %}
20667 ins_encode %{
20668 int opcode = this->ideal_Opcode();
20669 int vlen_enc = vector_length_encoding(this);
20670 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20671
20672 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20673 %}
20674 ins_pipe( pipe_slow );
20675 %}
20676
20677 // Long vector Min/Max
20678 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
20679 predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
20680 UseAVX == 0);
20681 match(Set dst (MinV dst src));
20682 match(Set dst (MaxV src dst));
20683 effect(TEMP dst, TEMP tmp);
20684 format %{ "vector_minmaxL $dst,$src\t!using $tmp as TEMP" %}
20685 ins_encode %{
20686 assert(UseSSE >= 4, "required");
20687
20688 int opcode = this->ideal_Opcode();
20689 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20690 assert(elem_bt == T_LONG, "sanity");
20691
20692 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
20693 %}
20694 ins_pipe( pipe_slow );
20695 %}
20696
20697 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
20698 predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
20699 UseAVX > 0 && !VM_Version::supports_avx512vl());
20700 match(Set dst (MinV src1 src2));
20701 match(Set dst (MaxV src1 src2));
20702 effect(TEMP dst);
20703 format %{ "vector_minmaxL $dst,$src1,$src2\t! " %}
20704 ins_encode %{
20705 int vlen_enc = vector_length_encoding(this);
20706 int opcode = this->ideal_Opcode();
20707 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20708 assert(elem_bt == T_LONG, "sanity");
20709
20710 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20711 %}
20712 ins_pipe( pipe_slow );
20713 %}
20714
20715 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
20716 predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
20717 Matcher::vector_element_basic_type(n) == T_LONG);
20718 match(Set dst (MinV src1 src2));
20719 match(Set dst (MaxV src1 src2));
20720 format %{ "vector_minmaxL $dst,$src1,src2\t! " %}
20721 ins_encode %{
20722 assert(UseAVX > 2, "required");
20723
20724 int vlen_enc = vector_length_encoding(this);
20725 int opcode = this->ideal_Opcode();
20726 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20727 assert(elem_bt == T_LONG, "sanity");
20728
20729 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20730 %}
20731 ins_pipe( pipe_slow );
20732 %}
20733
20734 // Float/Double vector Min/Max
20735 instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
20736 predicate(VM_Version::supports_avx10_2() &&
20737 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20738 match(Set dst (MinV a b));
20739 match(Set dst (MaxV a b));
20740 format %{ "vector_minmaxFP $dst, $a, $b" %}
20741 ins_encode %{
20742 int vlen_enc = vector_length_encoding(this);
20743 int opcode = this->ideal_Opcode();
20744 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20745 __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20746 %}
20747 ins_pipe( pipe_slow );
20748 %}
20749
20750 // Float/Double vector Min/Max
20751 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
20752 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) <= 32 &&
20753 is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
20754 UseAVX > 0);
20755 match(Set dst (MinV a b));
20756 match(Set dst (MaxV a b));
20757 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
20758 format %{ "vector_minmaxFP $dst,$a,$b\t!using $tmp, $atmp, $btmp as TEMP" %}
20759 ins_encode %{
20760 assert(UseAVX > 0, "required");
20761
20762 int opcode = this->ideal_Opcode();
20763 int vlen_enc = vector_length_encoding(this);
20764 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20765
20766 __ vminmax_fp(opcode, elem_bt,
20767 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20768 $tmp$$XMMRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20769 %}
20770 ins_pipe( pipe_slow );
20771 %}
20772
20773 instruct evminmaxFP_reg_evex(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
20774 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) == 64 &&
20775 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20776 match(Set dst (MinV a b));
20777 match(Set dst (MaxV a b));
20778 effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
20779 format %{ "vector_minmaxFP $dst,$a,$b\t!using $atmp, $btmp as TEMP" %}
20780 ins_encode %{
20781 assert(UseAVX > 2, "required");
20782
20783 int opcode = this->ideal_Opcode();
20784 int vlen_enc = vector_length_encoding(this);
20785 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20786
20787 __ evminmax_fp(opcode, elem_bt,
20788 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20789 $ktmp$$KRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20790 %}
20791 ins_pipe( pipe_slow );
20792 %}
20793
20794 // ------------------------------ Unsigned vector Min/Max ----------------------
20795
20796 instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
20797 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20798 match(Set dst (UMinV a b));
20799 match(Set dst (UMaxV a b));
20800 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20801 ins_encode %{
20802 int opcode = this->ideal_Opcode();
20803 int vlen_enc = vector_length_encoding(this);
20804 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20805 assert(is_integral_type(elem_bt), "");
20806 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20807 %}
20808 ins_pipe( pipe_slow );
20809 %}
20810
20811 instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
20812 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20813 match(Set dst (UMinV a (LoadVector b)));
20814 match(Set dst (UMaxV a (LoadVector b)));
20815 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20816 ins_encode %{
20817 int opcode = this->ideal_Opcode();
20818 int vlen_enc = vector_length_encoding(this);
20819 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20820 assert(is_integral_type(elem_bt), "");
20821 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
20822 %}
20823 ins_pipe( pipe_slow );
20824 %}
20825
20826 instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
20827 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
20828 match(Set dst (UMinV a b));
20829 match(Set dst (UMaxV a b));
20830 effect(TEMP xtmp1, TEMP xtmp2);
20831 format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
20832 ins_encode %{
20833 int opcode = this->ideal_Opcode();
20834 int vlen_enc = vector_length_encoding(this);
20835 __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20836 %}
20837 ins_pipe( pipe_slow );
20838 %}
20839
20840 instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
20841 match(Set dst (UMinV (Binary dst src2) mask));
20842 match(Set dst (UMaxV (Binary dst src2) mask));
20843 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20844 ins_encode %{
20845 int vlen_enc = vector_length_encoding(this);
20846 BasicType bt = Matcher::vector_element_basic_type(this);
20847 int opc = this->ideal_Opcode();
20848 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20849 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
20850 %}
20851 ins_pipe( pipe_slow );
20852 %}
20853
20854 instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
20855 match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
20856 match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
20857 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20858 ins_encode %{
20859 int vlen_enc = vector_length_encoding(this);
20860 BasicType bt = Matcher::vector_element_basic_type(this);
20861 int opc = this->ideal_Opcode();
20862 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20863 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
20864 %}
20865 ins_pipe( pipe_slow );
20866 %}
20867
20868 // --------------------------------- Signum/CopySign ---------------------------
20869
20870 instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
20871 match(Set dst (SignumF dst (Binary zero one)));
20872 effect(KILL cr);
20873 format %{ "signumF $dst, $dst" %}
20874 ins_encode %{
20875 int opcode = this->ideal_Opcode();
20876 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20877 %}
20878 ins_pipe( pipe_slow );
20879 %}
20880
20881 instruct signumD_reg(regD dst, regD zero, regD one, rFlagsReg cr) %{
20882 match(Set dst (SignumD dst (Binary zero one)));
20883 effect(KILL cr);
20884 format %{ "signumD $dst, $dst" %}
20885 ins_encode %{
20886 int opcode = this->ideal_Opcode();
20887 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20888 %}
20889 ins_pipe( pipe_slow );
20890 %}
20891
20892 instruct signumV_reg_avx(vec dst, vec src, vec zero, vec one, vec xtmp1) %{
20893 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
20894 match(Set dst (SignumVF src (Binary zero one)));
20895 match(Set dst (SignumVD src (Binary zero one)));
20896 effect(TEMP dst, TEMP xtmp1);
20897 format %{ "vector_signum_avx $dst, $src\t! using $xtmp1 as TEMP" %}
20898 ins_encode %{
20899 int opcode = this->ideal_Opcode();
20900 int vec_enc = vector_length_encoding(this);
20901 __ vector_signum_avx(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20902 $xtmp1$$XMMRegister, vec_enc);
20903 %}
20904 ins_pipe( pipe_slow );
20905 %}
20906
20907 instruct signumV_reg_evex(vec dst, vec src, vec zero, vec one, kReg ktmp1) %{
20908 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
20909 match(Set dst (SignumVF src (Binary zero one)));
20910 match(Set dst (SignumVD src (Binary zero one)));
20911 effect(TEMP dst, TEMP ktmp1);
20912 format %{ "vector_signum_evex $dst, $src\t! using $ktmp1 as TEMP" %}
20913 ins_encode %{
20914 int opcode = this->ideal_Opcode();
20915 int vec_enc = vector_length_encoding(this);
20916 __ vector_signum_evex(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20917 $ktmp1$$KRegister, vec_enc);
20918 %}
20919 ins_pipe( pipe_slow );
20920 %}
20921
20922 // ---------------------------------------
20923 // For copySign use 0xE4 as writemask for vpternlog
20924 // Desired Truth Table: A -> xmm0 bit, B -> xmm1 bit, C -> xmm2 bit
20925 // C (xmm2) is set to 0x7FFFFFFF
20926 // Wherever xmm2 is 0, we want to pick from B (sign)
20927 // Wherever xmm2 is 1, we want to pick from A (src)
20928 //
20929 // A B C Result
20930 // 0 0 0 0
20931 // 0 0 1 0
20932 // 0 1 0 1
20933 // 0 1 1 0
20934 // 1 0 0 0
20935 // 1 0 1 1
20936 // 1 1 0 1
20937 // 1 1 1 1
20938 //
20939 // Result going from high bit to low bit is 0x11100100 = 0xe4
20940 // ---------------------------------------
20941
20942 instruct copySignF_reg(regF dst, regF src, regF tmp1, rRegI tmp2) %{
20943 match(Set dst (CopySignF dst src));
20944 effect(TEMP tmp1, TEMP tmp2);
20945 format %{ "CopySignF $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20946 ins_encode %{
20947 __ movl($tmp2$$Register, 0x7FFFFFFF);
20948 __ movdl($tmp1$$XMMRegister, $tmp2$$Register);
20949 __ vpternlogd($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20950 %}
20951 ins_pipe( pipe_slow );
20952 %}
20953
20954 instruct copySignD_imm(regD dst, regD src, regD tmp1, rRegL tmp2, immD zero) %{
20955 match(Set dst (CopySignD dst (Binary src zero)));
20956 ins_cost(100);
20957 effect(TEMP tmp1, TEMP tmp2);
20958 format %{ "CopySignD $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20959 ins_encode %{
20960 __ mov64($tmp2$$Register, 0x7FFFFFFFFFFFFFFF);
20961 __ movq($tmp1$$XMMRegister, $tmp2$$Register);
20962 __ vpternlogq($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20963 %}
20964 ins_pipe( pipe_slow );
20965 %}
20966
20967 //----------------------------- CompressBits/ExpandBits ------------------------
20968
20969 instruct compressBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20970 predicate(n->bottom_type()->isa_int());
20971 match(Set dst (CompressBits src mask));
20972 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20973 ins_encode %{
20974 __ pextl($dst$$Register, $src$$Register, $mask$$Register);
20975 %}
20976 ins_pipe( pipe_slow );
20977 %}
20978
20979 instruct expandBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20980 predicate(n->bottom_type()->isa_int());
20981 match(Set dst (ExpandBits src mask));
20982 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20983 ins_encode %{
20984 __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
20985 %}
20986 ins_pipe( pipe_slow );
20987 %}
20988
20989 instruct compressBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20990 predicate(n->bottom_type()->isa_int());
20991 match(Set dst (CompressBits src (LoadI mask)));
20992 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20993 ins_encode %{
20994 __ pextl($dst$$Register, $src$$Register, $mask$$Address);
20995 %}
20996 ins_pipe( pipe_slow );
20997 %}
20998
20999 instruct expandBitsI_mem(rRegI dst, rRegI src, memory mask) %{
21000 predicate(n->bottom_type()->isa_int());
21001 match(Set dst (ExpandBits src (LoadI mask)));
21002 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
21003 ins_encode %{
21004 __ pdepl($dst$$Register, $src$$Register, $mask$$Address);
21005 %}
21006 ins_pipe( pipe_slow );
21007 %}
21008
21009 // --------------------------------- Sqrt --------------------------------------
21010
21011 instruct vsqrtF_reg(vec dst, vec src) %{
21012 match(Set dst (SqrtVF src));
21013 format %{ "vsqrtps $dst,$src\t! sqrt packedF" %}
21014 ins_encode %{
21015 assert(UseAVX > 0, "required");
21016 int vlen_enc = vector_length_encoding(this);
21017 __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21018 %}
21019 ins_pipe( pipe_slow );
21020 %}
21021
21022 instruct vsqrtF_mem(vec dst, memory mem) %{
21023 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
21024 match(Set dst (SqrtVF (LoadVector mem)));
21025 format %{ "vsqrtps $dst,$mem\t! sqrt packedF" %}
21026 ins_encode %{
21027 assert(UseAVX > 0, "required");
21028 int vlen_enc = vector_length_encoding(this);
21029 __ vsqrtps($dst$$XMMRegister, $mem$$Address, vlen_enc);
21030 %}
21031 ins_pipe( pipe_slow );
21032 %}
21033
21034 // Floating point vector sqrt
21035 instruct vsqrtD_reg(vec dst, vec src) %{
21036 match(Set dst (SqrtVD src));
21037 format %{ "vsqrtpd $dst,$src\t! sqrt packedD" %}
21038 ins_encode %{
21039 assert(UseAVX > 0, "required");
21040 int vlen_enc = vector_length_encoding(this);
21041 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21042 %}
21043 ins_pipe( pipe_slow );
21044 %}
21045
21046 instruct vsqrtD_mem(vec dst, memory mem) %{
21047 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
21048 match(Set dst (SqrtVD (LoadVector mem)));
21049 format %{ "vsqrtpd $dst,$mem\t! sqrt packedD" %}
21050 ins_encode %{
21051 assert(UseAVX > 0, "required");
21052 int vlen_enc = vector_length_encoding(this);
21053 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vlen_enc);
21054 %}
21055 ins_pipe( pipe_slow );
21056 %}
21057
21058 // ------------------------------ Shift ---------------------------------------
21059
21060 // Left and right shift count vectors are the same on x86
21061 // (only lowest bits of xmm reg are used for count).
21062 instruct vshiftcnt(vec dst, rRegI cnt) %{
21063 match(Set dst (LShiftCntV cnt));
21064 match(Set dst (RShiftCntV cnt));
21065 format %{ "movdl $dst,$cnt\t! load shift count" %}
21066 ins_encode %{
21067 __ movdl($dst$$XMMRegister, $cnt$$Register);
21068 %}
21069 ins_pipe( pipe_slow );
21070 %}
21071
21072 // Byte vector shift
21073 instruct vshiftB(vec dst, vec src, vec shift, vec tmp) %{
21074 predicate(Matcher::vector_length(n) <= 8 && !n->as_ShiftV()->is_var_shift());
21075 match(Set dst ( LShiftVB src shift));
21076 match(Set dst ( RShiftVB src shift));
21077 match(Set dst (URShiftVB src shift));
21078 effect(TEMP dst, USE src, USE shift, TEMP tmp);
21079 format %{"vector_byte_shift $dst,$src,$shift" %}
21080 ins_encode %{
21081 assert(UseSSE > 3, "required");
21082 int opcode = this->ideal_Opcode();
21083 bool sign = (opcode != Op_URShiftVB);
21084 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister);
21085 __ vshiftw(opcode, $tmp$$XMMRegister, $shift$$XMMRegister);
21086 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21087 __ pand($dst$$XMMRegister, $tmp$$XMMRegister);
21088 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
21089 %}
21090 ins_pipe( pipe_slow );
21091 %}
21092
21093 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
21094 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
21095 UseAVX <= 1);
21096 match(Set dst ( LShiftVB src shift));
21097 match(Set dst ( RShiftVB src shift));
21098 match(Set dst (URShiftVB src shift));
21099 effect(TEMP dst, USE src, USE shift, TEMP tmp1, TEMP tmp2);
21100 format %{"vector_byte_shift $dst,$src,$shift" %}
21101 ins_encode %{
21102 assert(UseSSE > 3, "required");
21103 int opcode = this->ideal_Opcode();
21104 bool sign = (opcode != Op_URShiftVB);
21105 __ vextendbw(sign, $tmp1$$XMMRegister, $src$$XMMRegister);
21106 __ vshiftw(opcode, $tmp1$$XMMRegister, $shift$$XMMRegister);
21107 __ pshufd($tmp2$$XMMRegister, $src$$XMMRegister, 0xE);
21108 __ vextendbw(sign, $tmp2$$XMMRegister, $tmp2$$XMMRegister);
21109 __ vshiftw(opcode, $tmp2$$XMMRegister, $shift$$XMMRegister);
21110 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21111 __ pand($tmp2$$XMMRegister, $dst$$XMMRegister);
21112 __ pand($dst$$XMMRegister, $tmp1$$XMMRegister);
21113 __ packuswb($dst$$XMMRegister, $tmp2$$XMMRegister);
21114 %}
21115 ins_pipe( pipe_slow );
21116 %}
21117
21118 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp) %{
21119 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
21120 UseAVX > 1);
21121 match(Set dst ( LShiftVB src shift));
21122 match(Set dst ( RShiftVB src shift));
21123 match(Set dst (URShiftVB src shift));
21124 effect(TEMP dst, TEMP tmp);
21125 format %{"vector_byte_shift $dst,$src,$shift" %}
21126 ins_encode %{
21127 int opcode = this->ideal_Opcode();
21128 bool sign = (opcode != Op_URShiftVB);
21129 int vlen_enc = Assembler::AVX_256bit;
21130 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
21131 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21132 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21133 __ vextracti128_high($dst$$XMMRegister, $tmp$$XMMRegister);
21134 __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $dst$$XMMRegister, 0);
21135 %}
21136 ins_pipe( pipe_slow );
21137 %}
21138
21139 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp) %{
21140 predicate(Matcher::vector_length(n) == 32 && !n->as_ShiftV()->is_var_shift());
21141 match(Set dst ( LShiftVB src shift));
21142 match(Set dst ( RShiftVB src shift));
21143 match(Set dst (URShiftVB src shift));
21144 effect(TEMP dst, TEMP tmp);
21145 format %{"vector_byte_shift $dst,$src,$shift" %}
21146 ins_encode %{
21147 assert(UseAVX > 1, "required");
21148 int opcode = this->ideal_Opcode();
21149 bool sign = (opcode != Op_URShiftVB);
21150 int vlen_enc = Assembler::AVX_256bit;
21151 __ vextracti128_high($tmp$$XMMRegister, $src$$XMMRegister);
21152 __ vextendbw(sign, $tmp$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21153 __ vextendbw(sign, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21154 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21155 __ vshiftw(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21156 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21157 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21158 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21159 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21160 %}
21161 ins_pipe( pipe_slow );
21162 %}
21163
21164 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
21165 predicate(Matcher::vector_length(n) == 64 && !n->as_ShiftV()->is_var_shift());
21166 match(Set dst ( LShiftVB src shift));
21167 match(Set dst (RShiftVB src shift));
21168 match(Set dst (URShiftVB src shift));
21169 effect(TEMP dst, TEMP tmp1, TEMP tmp2);
21170 format %{"vector_byte_shift $dst,$src,$shift" %}
21171 ins_encode %{
21172 assert(UseAVX > 2, "required");
21173 int opcode = this->ideal_Opcode();
21174 bool sign = (opcode != Op_URShiftVB);
21175 int vlen_enc = Assembler::AVX_512bit;
21176 __ vextracti64x4($tmp1$$XMMRegister, $src$$XMMRegister, 1);
21177 __ vextendbw(sign, $tmp1$$XMMRegister, $tmp1$$XMMRegister, vlen_enc);
21178 __ vextendbw(sign, $tmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21179 __ vshiftw(opcode, $tmp1$$XMMRegister, $tmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21180 __ vshiftw(opcode, $tmp2$$XMMRegister, $tmp2$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21181 __ vmovdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21182 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21183 __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21184 __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21185 __ vpackuswb($dst$$XMMRegister, $tmp1$$XMMRegister, $tmp2$$XMMRegister, vlen_enc);
21186 __ evmovdquq($tmp2$$XMMRegister, ExternalAddress(vector_byte_perm_mask()), vlen_enc, noreg);
21187 __ vpermq($dst$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21188 %}
21189 ins_pipe( pipe_slow );
21190 %}
21191
21192 // Shorts vector logical right shift produces incorrect Java result
21193 // for negative data because java code convert short value into int with
21194 // sign extension before a shift. But char vectors are fine since chars are
21195 // unsigned values.
21196 // Shorts/Chars vector left shift
21197 instruct vshiftS(vec dst, vec src, vec shift) %{
21198 predicate(!n->as_ShiftV()->is_var_shift());
21199 match(Set dst ( LShiftVS src shift));
21200 match(Set dst ( RShiftVS src shift));
21201 match(Set dst (URShiftVS src shift));
21202 effect(TEMP dst, USE src, USE shift);
21203 format %{ "vshiftw $dst,$src,$shift\t! shift packedS" %}
21204 ins_encode %{
21205 int opcode = this->ideal_Opcode();
21206 if (UseAVX > 0) {
21207 int vlen_enc = vector_length_encoding(this);
21208 __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21209 } else {
21210 int vlen = Matcher::vector_length(this);
21211 if (vlen == 2) {
21212 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21213 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21214 } else if (vlen == 4) {
21215 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21216 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21217 } else {
21218 assert (vlen == 8, "sanity");
21219 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21220 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21221 }
21222 }
21223 %}
21224 ins_pipe( pipe_slow );
21225 %}
21226
21227 // Integers vector left shift
21228 instruct vshiftI(vec dst, vec src, vec shift) %{
21229 predicate(!n->as_ShiftV()->is_var_shift());
21230 match(Set dst ( LShiftVI src shift));
21231 match(Set dst ( RShiftVI src shift));
21232 match(Set dst (URShiftVI src shift));
21233 effect(TEMP dst, USE src, USE shift);
21234 format %{ "vshiftd $dst,$src,$shift\t! shift packedI" %}
21235 ins_encode %{
21236 int opcode = this->ideal_Opcode();
21237 if (UseAVX > 0) {
21238 int vlen_enc = vector_length_encoding(this);
21239 __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21240 } else {
21241 int vlen = Matcher::vector_length(this);
21242 if (vlen == 2) {
21243 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21244 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21245 } else {
21246 assert(vlen == 4, "sanity");
21247 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21248 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21249 }
21250 }
21251 %}
21252 ins_pipe( pipe_slow );
21253 %}
21254
21255 // Integers vector left constant shift
21256 instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
21257 match(Set dst (LShiftVI src (LShiftCntV shift)));
21258 match(Set dst (RShiftVI src (RShiftCntV shift)));
21259 match(Set dst (URShiftVI src (RShiftCntV shift)));
21260 format %{ "vshiftd_imm $dst,$src,$shift\t! shift packedI" %}
21261 ins_encode %{
21262 int opcode = this->ideal_Opcode();
21263 if (UseAVX > 0) {
21264 int vector_len = vector_length_encoding(this);
21265 __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
21266 } else {
21267 int vlen = Matcher::vector_length(this);
21268 if (vlen == 2) {
21269 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21270 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21271 } else {
21272 assert(vlen == 4, "sanity");
21273 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21274 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21275 }
21276 }
21277 %}
21278 ins_pipe( pipe_slow );
21279 %}
21280
21281 // Longs vector shift
21282 instruct vshiftL(vec dst, vec src, vec shift) %{
21283 predicate(!n->as_ShiftV()->is_var_shift());
21284 match(Set dst ( LShiftVL src shift));
21285 match(Set dst (URShiftVL src shift));
21286 effect(TEMP dst, USE src, USE shift);
21287 format %{ "vshiftq $dst,$src,$shift\t! shift packedL" %}
21288 ins_encode %{
21289 int opcode = this->ideal_Opcode();
21290 if (UseAVX > 0) {
21291 int vlen_enc = vector_length_encoding(this);
21292 __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21293 } else {
21294 assert(Matcher::vector_length(this) == 2, "");
21295 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21296 __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21297 }
21298 %}
21299 ins_pipe( pipe_slow );
21300 %}
21301
21302 // Longs vector constant shift
21303 instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
21304 match(Set dst (LShiftVL src (LShiftCntV shift)));
21305 match(Set dst (URShiftVL src (RShiftCntV shift)));
21306 format %{ "vshiftq_imm $dst,$src,$shift\t! shift packedL" %}
21307 ins_encode %{
21308 int opcode = this->ideal_Opcode();
21309 if (UseAVX > 0) {
21310 int vector_len = vector_length_encoding(this);
21311 __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
21312 } else {
21313 assert(Matcher::vector_length(this) == 2, "");
21314 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21315 __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21316 }
21317 %}
21318 ins_pipe( pipe_slow );
21319 %}
21320
21321 // -------------------ArithmeticRightShift -----------------------------------
21322 // Long vector arithmetic right shift
21323 instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp) %{
21324 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX <= 2);
21325 match(Set dst (RShiftVL src shift));
21326 effect(TEMP dst, TEMP tmp);
21327 format %{ "vshiftq $dst,$src,$shift" %}
21328 ins_encode %{
21329 uint vlen = Matcher::vector_length(this);
21330 if (vlen == 2) {
21331 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21332 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
21333 __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21334 __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
21335 __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
21336 __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
21337 } else {
21338 assert(vlen == 4, "sanity");
21339 assert(UseAVX > 1, "required");
21340 int vlen_enc = Assembler::AVX_256bit;
21341 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21342 __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21343 __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21344 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21345 __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21346 }
21347 %}
21348 ins_pipe( pipe_slow );
21349 %}
21350
21351 instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
21352 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX > 2);
21353 match(Set dst (RShiftVL src shift));
21354 format %{ "vshiftq $dst,$src,$shift" %}
21355 ins_encode %{
21356 int vlen_enc = vector_length_encoding(this);
21357 __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21358 %}
21359 ins_pipe( pipe_slow );
21360 %}
21361
21362 // ------------------- Variable Shift -----------------------------
21363 // Byte variable shift
21364 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21365 predicate(Matcher::vector_length(n) <= 8 &&
21366 n->as_ShiftV()->is_var_shift() &&
21367 !VM_Version::supports_avx512bw());
21368 match(Set dst ( LShiftVB src shift));
21369 match(Set dst ( RShiftVB src shift));
21370 match(Set dst (URShiftVB src shift));
21371 effect(TEMP dst, TEMP vtmp);
21372 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21373 ins_encode %{
21374 assert(UseAVX >= 2, "required");
21375
21376 int opcode = this->ideal_Opcode();
21377 int vlen_enc = Assembler::AVX_128bit;
21378 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21379 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21380 %}
21381 ins_pipe( pipe_slow );
21382 %}
21383
21384 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21385 predicate(Matcher::vector_length(n) == 16 &&
21386 n->as_ShiftV()->is_var_shift() &&
21387 !VM_Version::supports_avx512bw());
21388 match(Set dst ( LShiftVB src shift));
21389 match(Set dst ( RShiftVB src shift));
21390 match(Set dst (URShiftVB src shift));
21391 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21392 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21393 ins_encode %{
21394 assert(UseAVX >= 2, "required");
21395
21396 int opcode = this->ideal_Opcode();
21397 int vlen_enc = Assembler::AVX_128bit;
21398 // Shift lower half and get word result in dst
21399 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21400
21401 // Shift upper half and get word result in vtmp1
21402 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21403 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21404 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21405
21406 // Merge and down convert the two word results to byte in dst
21407 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21408 %}
21409 ins_pipe( pipe_slow );
21410 %}
21411
21412 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4) %{
21413 predicate(Matcher::vector_length(n) == 32 &&
21414 n->as_ShiftV()->is_var_shift() &&
21415 !VM_Version::supports_avx512bw());
21416 match(Set dst ( LShiftVB src shift));
21417 match(Set dst ( RShiftVB src shift));
21418 match(Set dst (URShiftVB src shift));
21419 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4);
21420 format %{ "vector_varshift_byte $dst, $src, $shift\n\t using $vtmp1, $vtmp2, $vtmp3, $vtmp4 as TEMP" %}
21421 ins_encode %{
21422 assert(UseAVX >= 2, "required");
21423
21424 int opcode = this->ideal_Opcode();
21425 int vlen_enc = Assembler::AVX_128bit;
21426 // Process lower 128 bits and get result in dst
21427 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21428 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21429 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21430 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21431 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21432
21433 // Process higher 128 bits and get result in vtmp3
21434 __ vextracti128_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21435 __ vextracti128_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21436 __ varshiftbw(opcode, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp4$$XMMRegister);
21437 __ vpshufd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, 0xE, 0);
21438 __ vpshufd($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, 0xE, 0);
21439 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21440 __ vpackuswb($vtmp1$$XMMRegister, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, 0);
21441
21442 // Merge the two results in dst
21443 __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21444 %}
21445 ins_pipe( pipe_slow );
21446 %}
21447
21448 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp) %{
21449 predicate(Matcher::vector_length(n) <= 32 &&
21450 n->as_ShiftV()->is_var_shift() &&
21451 VM_Version::supports_avx512bw());
21452 match(Set dst ( LShiftVB src shift));
21453 match(Set dst ( RShiftVB src shift));
21454 match(Set dst (URShiftVB src shift));
21455 effect(TEMP dst, TEMP vtmp);
21456 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21457 ins_encode %{
21458 assert(UseAVX > 2, "required");
21459
21460 int opcode = this->ideal_Opcode();
21461 int vlen_enc = vector_length_encoding(this);
21462 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21463 %}
21464 ins_pipe( pipe_slow );
21465 %}
21466
21467 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21468 predicate(Matcher::vector_length(n) == 64 &&
21469 n->as_ShiftV()->is_var_shift() &&
21470 VM_Version::supports_avx512bw());
21471 match(Set dst ( LShiftVB src shift));
21472 match(Set dst ( RShiftVB src shift));
21473 match(Set dst (URShiftVB src shift));
21474 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21475 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21476 ins_encode %{
21477 assert(UseAVX > 2, "required");
21478
21479 int opcode = this->ideal_Opcode();
21480 int vlen_enc = Assembler::AVX_256bit;
21481 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21482 __ vextracti64x4_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21483 __ vextracti64x4_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21484 __ evarshiftb(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21485 __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21486 %}
21487 ins_pipe( pipe_slow );
21488 %}
21489
21490 // Short variable shift
21491 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21492 predicate(Matcher::vector_length(n) <= 8 &&
21493 n->as_ShiftV()->is_var_shift() &&
21494 !VM_Version::supports_avx512bw());
21495 match(Set dst ( LShiftVS src shift));
21496 match(Set dst ( RShiftVS src shift));
21497 match(Set dst (URShiftVS src shift));
21498 effect(TEMP dst, TEMP vtmp);
21499 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21500 ins_encode %{
21501 assert(UseAVX >= 2, "required");
21502
21503 int opcode = this->ideal_Opcode();
21504 bool sign = (opcode != Op_URShiftVS);
21505 int vlen_enc = Assembler::AVX_256bit;
21506 __ vextendwd(sign, $dst$$XMMRegister, $src$$XMMRegister, 1);
21507 __ vpmovzxwd($vtmp$$XMMRegister, $shift$$XMMRegister, 1);
21508 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
21509 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21510 __ vextracti128_high($vtmp$$XMMRegister, $dst$$XMMRegister);
21511 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21512 %}
21513 ins_pipe( pipe_slow );
21514 %}
21515
21516 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21517 predicate(Matcher::vector_length(n) == 16 &&
21518 n->as_ShiftV()->is_var_shift() &&
21519 !VM_Version::supports_avx512bw());
21520 match(Set dst ( LShiftVS src shift));
21521 match(Set dst ( RShiftVS src shift));
21522 match(Set dst (URShiftVS src shift));
21523 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21524 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21525 ins_encode %{
21526 assert(UseAVX >= 2, "required");
21527
21528 int opcode = this->ideal_Opcode();
21529 bool sign = (opcode != Op_URShiftVS);
21530 int vlen_enc = Assembler::AVX_256bit;
21531 // Shift lower half, with result in vtmp2 using vtmp1 as TEMP
21532 __ vextendwd(sign, $vtmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21533 __ vpmovzxwd($vtmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21534 __ varshiftd(opcode, $vtmp2$$XMMRegister, $vtmp2$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21535 __ vpand($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21536
21537 // Shift upper half, with result in dst using vtmp1 as TEMP
21538 __ vextracti128_high($dst$$XMMRegister, $src$$XMMRegister);
21539 __ vextracti128_high($vtmp1$$XMMRegister, $shift$$XMMRegister);
21540 __ vextendwd(sign, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21541 __ vpmovzxwd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21542 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21543 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21544
21545 // Merge lower and upper half result into dst
21546 __ vpackusdw($dst$$XMMRegister, $vtmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21547 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21548 %}
21549 ins_pipe( pipe_slow );
21550 %}
21551
21552 instruct vshift16S_var_evex_bw(vec dst, vec src, vec shift) %{
21553 predicate(n->as_ShiftV()->is_var_shift() &&
21554 VM_Version::supports_avx512bw());
21555 match(Set dst ( LShiftVS src shift));
21556 match(Set dst ( RShiftVS src shift));
21557 match(Set dst (URShiftVS src shift));
21558 format %{ "vector_varshift_short $dst,$src,$shift\t!" %}
21559 ins_encode %{
21560 assert(UseAVX > 2, "required");
21561
21562 int opcode = this->ideal_Opcode();
21563 int vlen_enc = vector_length_encoding(this);
21564 if (!VM_Version::supports_avx512vl()) {
21565 vlen_enc = Assembler::AVX_512bit;
21566 }
21567 __ varshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21568 %}
21569 ins_pipe( pipe_slow );
21570 %}
21571
21572 //Integer variable shift
21573 instruct vshiftI_var(vec dst, vec src, vec shift) %{
21574 predicate(n->as_ShiftV()->is_var_shift());
21575 match(Set dst ( LShiftVI src shift));
21576 match(Set dst ( RShiftVI src shift));
21577 match(Set dst (URShiftVI src shift));
21578 format %{ "vector_varshift_int $dst,$src,$shift\t!" %}
21579 ins_encode %{
21580 assert(UseAVX >= 2, "required");
21581
21582 int opcode = this->ideal_Opcode();
21583 int vlen_enc = vector_length_encoding(this);
21584 __ varshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21585 %}
21586 ins_pipe( pipe_slow );
21587 %}
21588
21589 //Long variable shift
21590 instruct vshiftL_var(vec dst, vec src, vec shift) %{
21591 predicate(n->as_ShiftV()->is_var_shift());
21592 match(Set dst ( LShiftVL src shift));
21593 match(Set dst (URShiftVL src shift));
21594 format %{ "vector_varshift_long $dst,$src,$shift\t!" %}
21595 ins_encode %{
21596 assert(UseAVX >= 2, "required");
21597
21598 int opcode = this->ideal_Opcode();
21599 int vlen_enc = vector_length_encoding(this);
21600 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21601 %}
21602 ins_pipe( pipe_slow );
21603 %}
21604
21605 //Long variable right shift arithmetic
21606 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
21607 predicate(Matcher::vector_length(n) <= 4 &&
21608 n->as_ShiftV()->is_var_shift() &&
21609 UseAVX == 2);
21610 match(Set dst (RShiftVL src shift));
21611 effect(TEMP dst, TEMP vtmp);
21612 format %{ "vector_varshift_long $dst,$src,$shift\n\t! using $vtmp as TEMP" %}
21613 ins_encode %{
21614 int opcode = this->ideal_Opcode();
21615 int vlen_enc = vector_length_encoding(this);
21616 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc,
21617 $vtmp$$XMMRegister);
21618 %}
21619 ins_pipe( pipe_slow );
21620 %}
21621
21622 instruct vshiftL_arith_var_evex(vec dst, vec src, vec shift) %{
21623 predicate(n->as_ShiftV()->is_var_shift() &&
21624 UseAVX > 2);
21625 match(Set dst (RShiftVL src shift));
21626 format %{ "vector_varfshift_long $dst,$src,$shift\t!" %}
21627 ins_encode %{
21628 int opcode = this->ideal_Opcode();
21629 int vlen_enc = vector_length_encoding(this);
21630 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21631 %}
21632 ins_pipe( pipe_slow );
21633 %}
21634
21635 // --------------------------------- AND --------------------------------------
21636
21637 instruct vand(vec dst, vec src) %{
21638 predicate(UseAVX == 0);
21639 match(Set dst (AndV dst src));
21640 format %{ "pand $dst,$src\t! and vectors" %}
21641 ins_encode %{
21642 __ pand($dst$$XMMRegister, $src$$XMMRegister);
21643 %}
21644 ins_pipe( pipe_slow );
21645 %}
21646
21647 instruct vand_reg(vec dst, vec src1, vec src2) %{
21648 predicate(UseAVX > 0);
21649 match(Set dst (AndV src1 src2));
21650 format %{ "vpand $dst,$src1,$src2\t! and vectors" %}
21651 ins_encode %{
21652 int vlen_enc = vector_length_encoding(this);
21653 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21654 %}
21655 ins_pipe( pipe_slow );
21656 %}
21657
21658 instruct vand_mem(vec dst, vec src, memory mem) %{
21659 predicate((UseAVX > 0) &&
21660 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21661 match(Set dst (AndV src (LoadVector mem)));
21662 format %{ "vpand $dst,$src,$mem\t! and vectors" %}
21663 ins_encode %{
21664 int vlen_enc = vector_length_encoding(this);
21665 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21666 %}
21667 ins_pipe( pipe_slow );
21668 %}
21669
21670 // --------------------------------- OR ---------------------------------------
21671
21672 instruct vor(vec dst, vec src) %{
21673 predicate(UseAVX == 0);
21674 match(Set dst (OrV dst src));
21675 format %{ "por $dst,$src\t! or vectors" %}
21676 ins_encode %{
21677 __ por($dst$$XMMRegister, $src$$XMMRegister);
21678 %}
21679 ins_pipe( pipe_slow );
21680 %}
21681
21682 instruct vor_reg(vec dst, vec src1, vec src2) %{
21683 predicate(UseAVX > 0);
21684 match(Set dst (OrV src1 src2));
21685 format %{ "vpor $dst,$src1,$src2\t! or vectors" %}
21686 ins_encode %{
21687 int vlen_enc = vector_length_encoding(this);
21688 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21689 %}
21690 ins_pipe( pipe_slow );
21691 %}
21692
21693 instruct vor_mem(vec dst, vec src, memory mem) %{
21694 predicate((UseAVX > 0) &&
21695 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21696 match(Set dst (OrV src (LoadVector mem)));
21697 format %{ "vpor $dst,$src,$mem\t! or vectors" %}
21698 ins_encode %{
21699 int vlen_enc = vector_length_encoding(this);
21700 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21701 %}
21702 ins_pipe( pipe_slow );
21703 %}
21704
21705 // --------------------------------- XOR --------------------------------------
21706
21707 instruct vxor(vec dst, vec src) %{
21708 predicate(UseAVX == 0);
21709 match(Set dst (XorV dst src));
21710 format %{ "pxor $dst,$src\t! xor vectors" %}
21711 ins_encode %{
21712 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
21713 %}
21714 ins_pipe( pipe_slow );
21715 %}
21716
21717 instruct vxor_reg(vec dst, vec src1, vec src2) %{
21718 predicate(UseAVX > 0);
21719 match(Set dst (XorV src1 src2));
21720 format %{ "vpxor $dst,$src1,$src2\t! xor vectors" %}
21721 ins_encode %{
21722 int vlen_enc = vector_length_encoding(this);
21723 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21724 %}
21725 ins_pipe( pipe_slow );
21726 %}
21727
21728 instruct vxor_mem(vec dst, vec src, memory mem) %{
21729 predicate((UseAVX > 0) &&
21730 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21731 match(Set dst (XorV src (LoadVector mem)));
21732 format %{ "vpxor $dst,$src,$mem\t! xor vectors" %}
21733 ins_encode %{
21734 int vlen_enc = vector_length_encoding(this);
21735 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21736 %}
21737 ins_pipe( pipe_slow );
21738 %}
21739
21740 // --------------------------------- VectorCast --------------------------------------
21741
21742 instruct vcastBtoX(vec dst, vec src) %{
21743 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_DOUBLE);
21744 match(Set dst (VectorCastB2X src));
21745 format %{ "vector_cast_b2x $dst,$src\t!" %}
21746 ins_encode %{
21747 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21748 int vlen_enc = vector_length_encoding(this);
21749 __ vconvert_b2x(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21750 %}
21751 ins_pipe( pipe_slow );
21752 %}
21753
21754 instruct vcastBtoD(legVec dst, legVec src) %{
21755 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_DOUBLE);
21756 match(Set dst (VectorCastB2X src));
21757 format %{ "vector_cast_b2x $dst,$src\t!" %}
21758 ins_encode %{
21759 int vlen_enc = vector_length_encoding(this);
21760 __ vconvert_b2x(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21761 %}
21762 ins_pipe( pipe_slow );
21763 %}
21764
21765 instruct castStoX(vec dst, vec src) %{
21766 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21767 Matcher::vector_length(n->in(1)) <= 8 && // src
21768 Matcher::vector_element_basic_type(n) == T_BYTE);
21769 match(Set dst (VectorCastS2X src));
21770 format %{ "vector_cast_s2x $dst,$src" %}
21771 ins_encode %{
21772 assert(UseAVX > 0, "required");
21773
21774 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), 0, noreg);
21775 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21776 %}
21777 ins_pipe( pipe_slow );
21778 %}
21779
21780 instruct vcastStoX(vec dst, vec src, vec vtmp) %{
21781 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21782 Matcher::vector_length(n->in(1)) == 16 && // src
21783 Matcher::vector_element_basic_type(n) == T_BYTE);
21784 effect(TEMP dst, TEMP vtmp);
21785 match(Set dst (VectorCastS2X src));
21786 format %{ "vector_cast_s2x $dst,$src\t! using $vtmp as TEMP" %}
21787 ins_encode %{
21788 assert(UseAVX > 0, "required");
21789
21790 int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
21791 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21792 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
21793 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21794 %}
21795 ins_pipe( pipe_slow );
21796 %}
21797
21798 instruct vcastStoX_evex(vec dst, vec src) %{
21799 predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
21800 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21801 match(Set dst (VectorCastS2X src));
21802 format %{ "vector_cast_s2x $dst,$src\t!" %}
21803 ins_encode %{
21804 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21805 int src_vlen_enc = vector_length_encoding(this, $src);
21806 int vlen_enc = vector_length_encoding(this);
21807 switch (to_elem_bt) {
21808 case T_BYTE:
21809 if (!VM_Version::supports_avx512vl()) {
21810 vlen_enc = Assembler::AVX_512bit;
21811 }
21812 __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21813 break;
21814 case T_INT:
21815 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21816 break;
21817 case T_FLOAT:
21818 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21819 __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21820 break;
21821 case T_LONG:
21822 __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21823 break;
21824 case T_DOUBLE: {
21825 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
21826 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, mid_vlen_enc);
21827 __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21828 break;
21829 }
21830 default:
21831 ShouldNotReachHere();
21832 }
21833 %}
21834 ins_pipe( pipe_slow );
21835 %}
21836
21837 instruct castItoX(vec dst, vec src) %{
21838 predicate(UseAVX <= 2 &&
21839 (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
21840 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21841 match(Set dst (VectorCastI2X src));
21842 format %{ "vector_cast_i2x $dst,$src" %}
21843 ins_encode %{
21844 assert(UseAVX > 0, "required");
21845
21846 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21847 int vlen_enc = vector_length_encoding(this, $src);
21848
21849 if (to_elem_bt == T_BYTE) {
21850 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21851 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21852 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21853 } else {
21854 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21855 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21856 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21857 }
21858 %}
21859 ins_pipe( pipe_slow );
21860 %}
21861
21862 instruct vcastItoX(vec dst, vec src, vec vtmp) %{
21863 predicate(UseAVX <= 2 &&
21864 (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
21865 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21866 match(Set dst (VectorCastI2X src));
21867 format %{ "vector_cast_i2x $dst,$src\t! using $vtmp as TEMP" %}
21868 effect(TEMP dst, TEMP vtmp);
21869 ins_encode %{
21870 assert(UseAVX > 0, "required");
21871
21872 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21873 int vlen_enc = vector_length_encoding(this, $src);
21874
21875 if (to_elem_bt == T_BYTE) {
21876 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21877 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21878 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21879 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21880 } else {
21881 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21882 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21883 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21884 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21885 }
21886 %}
21887 ins_pipe( pipe_slow );
21888 %}
21889
21890 instruct vcastItoX_evex(vec dst, vec src) %{
21891 predicate(UseAVX > 2 ||
21892 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21893 match(Set dst (VectorCastI2X src));
21894 format %{ "vector_cast_i2x $dst,$src\t!" %}
21895 ins_encode %{
21896 assert(UseAVX > 0, "required");
21897
21898 BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
21899 int src_vlen_enc = vector_length_encoding(this, $src);
21900 int dst_vlen_enc = vector_length_encoding(this);
21901 switch (dst_elem_bt) {
21902 case T_BYTE:
21903 if (!VM_Version::supports_avx512vl()) {
21904 src_vlen_enc = Assembler::AVX_512bit;
21905 }
21906 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21907 break;
21908 case T_SHORT:
21909 if (!VM_Version::supports_avx512vl()) {
21910 src_vlen_enc = Assembler::AVX_512bit;
21911 }
21912 __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21913 break;
21914 case T_FLOAT:
21915 __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21916 break;
21917 case T_LONG:
21918 __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21919 break;
21920 case T_DOUBLE:
21921 __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21922 break;
21923 default:
21924 ShouldNotReachHere();
21925 }
21926 %}
21927 ins_pipe( pipe_slow );
21928 %}
21929
21930 instruct vcastLtoBS(vec dst, vec src) %{
21931 predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
21932 UseAVX <= 2);
21933 match(Set dst (VectorCastL2X src));
21934 format %{ "vector_cast_l2x $dst,$src" %}
21935 ins_encode %{
21936 assert(UseAVX > 0, "required");
21937
21938 int vlen = Matcher::vector_length_in_bytes(this, $src);
21939 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21940 AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
21941 : ExternalAddress(vector_int_to_short_mask());
21942 if (vlen <= 16) {
21943 __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_128bit);
21944 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21945 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21946 } else {
21947 assert(vlen <= 32, "required");
21948 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_256bit);
21949 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, Assembler::AVX_256bit);
21950 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21951 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21952 }
21953 if (to_elem_bt == T_BYTE) {
21954 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21955 }
21956 %}
21957 ins_pipe( pipe_slow );
21958 %}
21959
21960 instruct vcastLtoX_evex(vec dst, vec src) %{
21961 predicate(UseAVX > 2 ||
21962 (Matcher::vector_element_basic_type(n) == T_INT ||
21963 Matcher::vector_element_basic_type(n) == T_FLOAT ||
21964 Matcher::vector_element_basic_type(n) == T_DOUBLE));
21965 match(Set dst (VectorCastL2X src));
21966 format %{ "vector_cast_l2x $dst,$src\t!" %}
21967 ins_encode %{
21968 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21969 int vlen = Matcher::vector_length_in_bytes(this, $src);
21970 int vlen_enc = vector_length_encoding(this, $src);
21971 switch (to_elem_bt) {
21972 case T_BYTE:
21973 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21974 vlen_enc = Assembler::AVX_512bit;
21975 }
21976 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21977 break;
21978 case T_SHORT:
21979 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21980 vlen_enc = Assembler::AVX_512bit;
21981 }
21982 __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21983 break;
21984 case T_INT:
21985 if (vlen == 8) {
21986 if ($dst$$XMMRegister != $src$$XMMRegister) {
21987 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21988 }
21989 } else if (vlen == 16) {
21990 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
21991 } else if (vlen == 32) {
21992 if (UseAVX > 2) {
21993 if (!VM_Version::supports_avx512vl()) {
21994 vlen_enc = Assembler::AVX_512bit;
21995 }
21996 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21997 } else {
21998 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vlen_enc);
21999 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vlen_enc);
22000 }
22001 } else { // vlen == 64
22002 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22003 }
22004 break;
22005 case T_FLOAT:
22006 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
22007 __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22008 break;
22009 case T_DOUBLE:
22010 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
22011 __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22012 break;
22013
22014 default: assert(false, "%s", type2name(to_elem_bt));
22015 }
22016 %}
22017 ins_pipe( pipe_slow );
22018 %}
22019
22020 instruct vcastFtoD_reg(vec dst, vec src) %{
22021 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
22022 match(Set dst (VectorCastF2X src));
22023 format %{ "vector_cast_f2d $dst,$src\t!" %}
22024 ins_encode %{
22025 int vlen_enc = vector_length_encoding(this);
22026 __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22027 %}
22028 ins_pipe( pipe_slow );
22029 %}
22030
22031
22032 instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
22033 predicate(!VM_Version::supports_avx10_2() &&
22034 !VM_Version::supports_avx512vl() &&
22035 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
22036 type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
22037 is_integral_type(Matcher::vector_element_basic_type(n)));
22038 match(Set dst (VectorCastF2X src));
22039 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
22040 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
22041 ins_encode %{
22042 int vlen_enc = vector_length_encoding(this, $src);
22043 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22044 // JDK-8292878 removed the need for an explicit scratch register needed to load greater than
22045 // 32 bit addresses for register indirect addressing mode since stub constants
22046 // are part of code cache and there is a cap of 2G on ReservedCodeCacheSize currently.
22047 // However, targets are free to increase this limit, but having a large code cache size
22048 // greater than 2G looks unreasonable in practical scenario, on the hind side with given
22049 // cap we save a temporary register allocation which in limiting case can prevent
22050 // spilling in high register pressure blocks.
22051 __ vector_castF2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22052 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
22053 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22054 %}
22055 ins_pipe( pipe_slow );
22056 %}
22057
22058 instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22059 predicate(!VM_Version::supports_avx10_2() &&
22060 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
22061 is_integral_type(Matcher::vector_element_basic_type(n)));
22062 match(Set dst (VectorCastF2X src));
22063 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22064 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
22065 ins_encode %{
22066 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22067 if (to_elem_bt == T_LONG) {
22068 int vlen_enc = vector_length_encoding(this);
22069 __ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22070 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
22071 ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
22072 } else {
22073 int vlen_enc = vector_length_encoding(this, $src);
22074 __ vector_castF2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22075 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
22076 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22077 }
22078 %}
22079 ins_pipe( pipe_slow );
22080 %}
22081
22082 instruct castFtoX_reg_avx10_2(vec dst, vec src) %{
22083 predicate(VM_Version::supports_avx10_2() &&
22084 is_integral_type(Matcher::vector_element_basic_type(n)));
22085 match(Set dst (VectorCastF2X src));
22086 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
22087 ins_encode %{
22088 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22089 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
22090 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22091 %}
22092 ins_pipe( pipe_slow );
22093 %}
22094
22095 instruct castFtoX_mem_avx10_2(vec dst, memory src) %{
22096 predicate(VM_Version::supports_avx10_2() &&
22097 is_integral_type(Matcher::vector_element_basic_type(n)));
22098 match(Set dst (VectorCastF2X (LoadVector src)));
22099 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
22100 ins_encode %{
22101 int vlen = Matcher::vector_length(this);
22102 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22103 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
22104 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
22105 %}
22106 ins_pipe( pipe_slow );
22107 %}
22108
22109 instruct vcastDtoF_reg(vec dst, vec src) %{
22110 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
22111 match(Set dst (VectorCastD2X src));
22112 format %{ "vector_cast_d2x $dst,$src\t!" %}
22113 ins_encode %{
22114 int vlen_enc = vector_length_encoding(this, $src);
22115 __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22116 %}
22117 ins_pipe( pipe_slow );
22118 %}
22119
22120 instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
22121 predicate(!VM_Version::supports_avx10_2() &&
22122 !VM_Version::supports_avx512vl() &&
22123 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
22124 is_integral_type(Matcher::vector_element_basic_type(n)));
22125 match(Set dst (VectorCastD2X src));
22126 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
22127 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $xtmp5 as TEMP" %}
22128 ins_encode %{
22129 int vlen_enc = vector_length_encoding(this, $src);
22130 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22131 __ vector_castD2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22132 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, $xtmp5$$XMMRegister,
22133 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22134 %}
22135 ins_pipe( pipe_slow );
22136 %}
22137
22138 instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22139 predicate(!VM_Version::supports_avx10_2() &&
22140 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
22141 is_integral_type(Matcher::vector_element_basic_type(n)));
22142 match(Set dst (VectorCastD2X src));
22143 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22144 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
22145 ins_encode %{
22146 int vlen_enc = vector_length_encoding(this, $src);
22147 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22148 AddressLiteral signflip = VM_Version::supports_avx512dq() ? ExternalAddress(vector_double_signflip()) :
22149 ExternalAddress(vector_float_signflip());
22150 __ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22151 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister, signflip, noreg, vlen_enc);
22152 %}
22153 ins_pipe( pipe_slow );
22154 %}
22155
22156 instruct castDtoX_reg_avx10_2(vec dst, vec src) %{
22157 predicate(VM_Version::supports_avx10_2() &&
22158 is_integral_type(Matcher::vector_element_basic_type(n)));
22159 match(Set dst (VectorCastD2X src));
22160 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
22161 ins_encode %{
22162 int vlen_enc = vector_length_encoding(this, $src);
22163 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22164 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22165 %}
22166 ins_pipe( pipe_slow );
22167 %}
22168
22169 instruct castDtoX_mem_avx10_2(vec dst, memory src) %{
22170 predicate(VM_Version::supports_avx10_2() &&
22171 is_integral_type(Matcher::vector_element_basic_type(n)));
22172 match(Set dst (VectorCastD2X (LoadVector src)));
22173 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
22174 ins_encode %{
22175 int vlen = Matcher::vector_length(this);
22176 int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
22177 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22178 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
22179 %}
22180 ins_pipe( pipe_slow );
22181 %}
22182
22183 instruct vucast(vec dst, vec src) %{
22184 match(Set dst (VectorUCastB2X src));
22185 match(Set dst (VectorUCastS2X src));
22186 match(Set dst (VectorUCastI2X src));
22187 format %{ "vector_ucast $dst,$src\t!" %}
22188 ins_encode %{
22189 assert(UseAVX > 0, "required");
22190
22191 BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
22192 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22193 int vlen_enc = vector_length_encoding(this);
22194 __ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
22195 %}
22196 ins_pipe( pipe_slow );
22197 %}
22198
22199 instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
22200 predicate(!VM_Version::supports_avx512vl() &&
22201 Matcher::vector_length_in_bytes(n) < 64 &&
22202 Matcher::vector_element_basic_type(n) == T_INT);
22203 match(Set dst (RoundVF src));
22204 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
22205 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
22206 ins_encode %{
22207 int vlen_enc = vector_length_encoding(this);
22208 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22209 __ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
22210 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
22211 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
22212 %}
22213 ins_pipe( pipe_slow );
22214 %}
22215
22216 instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22217 predicate((VM_Version::supports_avx512vl() ||
22218 Matcher::vector_length_in_bytes(n) == 64) &&
22219 Matcher::vector_element_basic_type(n) == T_INT);
22220 match(Set dst (RoundVF src));
22221 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22222 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
22223 ins_encode %{
22224 int vlen_enc = vector_length_encoding(this);
22225 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22226 __ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
22227 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
22228 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
22229 %}
22230 ins_pipe( pipe_slow );
22231 %}
22232
22233 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22234 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
22235 match(Set dst (RoundVD src));
22236 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22237 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
22238 ins_encode %{
22239 int vlen_enc = vector_length_encoding(this);
22240 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22241 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
22242 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
22243 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
22244 %}
22245 ins_pipe( pipe_slow );
22246 %}
22247
22248 // --------------------------------- VectorMaskCmp --------------------------------------
22249
22250 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
22251 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22252 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
22253 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22254 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22255 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22256 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22257 ins_encode %{
22258 int vlen_enc = vector_length_encoding(this, $src1);
22259 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22260 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22261 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22262 } else {
22263 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22264 }
22265 %}
22266 ins_pipe( pipe_slow );
22267 %}
22268
22269 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22270 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
22271 n->bottom_type()->isa_pvectmask() == nullptr &&
22272 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22273 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22274 effect(TEMP ktmp);
22275 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22276 ins_encode %{
22277 int vlen_enc = Assembler::AVX_512bit;
22278 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22279 KRegister mask = k0; // The comparison itself is not being masked.
22280 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22281 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22282 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
22283 } else {
22284 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22285 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
22286 }
22287 %}
22288 ins_pipe( pipe_slow );
22289 %}
22290
22291 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
22292 predicate(n->bottom_type()->isa_pvectmask() &&
22293 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22294 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22295 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
22296 ins_encode %{
22297 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22298 int vlen_enc = vector_length_encoding(this, $src1);
22299 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22300 KRegister mask = k0; // The comparison itself is not being masked.
22301 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22302 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22303 } else {
22304 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22305 }
22306 %}
22307 ins_pipe( pipe_slow );
22308 %}
22309
22310 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
22311 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22312 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22313 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22314 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22315 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22316 (n->in(2)->get_int() == BoolTest::eq ||
22317 n->in(2)->get_int() == BoolTest::lt ||
22318 n->in(2)->get_int() == BoolTest::gt)); // cond
22319 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22320 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22321 ins_encode %{
22322 int vlen_enc = vector_length_encoding(this, $src1);
22323 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22324 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22325 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
22326 %}
22327 ins_pipe( pipe_slow );
22328 %}
22329
22330 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22331 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22332 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22333 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22334 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22335 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22336 (n->in(2)->get_int() == BoolTest::ne ||
22337 n->in(2)->get_int() == BoolTest::le ||
22338 n->in(2)->get_int() == BoolTest::ge)); // cond
22339 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22340 effect(TEMP dst, TEMP xtmp);
22341 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22342 ins_encode %{
22343 int vlen_enc = vector_length_encoding(this, $src1);
22344 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22345 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22346 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22347 %}
22348 ins_pipe( pipe_slow );
22349 %}
22350
22351 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22352 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22353 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22354 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22355 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22356 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22357 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22358 effect(TEMP dst, TEMP xtmp);
22359 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22360 ins_encode %{
22361 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
22362 int vlen_enc = vector_length_encoding(this, $src1);
22363 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22364 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22365
22366 if (vlen_enc == Assembler::AVX_128bit) {
22367 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22368 } else {
22369 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22370 }
22371 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22372 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22373 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22374 %}
22375 ins_pipe( pipe_slow );
22376 %}
22377
22378 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22379 predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
22380 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
22381 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22382 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22383 effect(TEMP ktmp);
22384 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22385 ins_encode %{
22386 assert(UseAVX > 2, "required");
22387
22388 int vlen_enc = vector_length_encoding(this, $src1);
22389 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22390 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22391 KRegister mask = k0; // The comparison itself is not being masked.
22392 bool merge = false;
22393 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22394
22395 switch (src1_elem_bt) {
22396 case T_INT: {
22397 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22398 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22399 break;
22400 }
22401 case T_LONG: {
22402 __ evpcmpq($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22403 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22404 break;
22405 }
22406 default: assert(false, "%s", type2name(src1_elem_bt));
22407 }
22408 %}
22409 ins_pipe( pipe_slow );
22410 %}
22411
22412
22413 instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
22414 predicate(n->bottom_type()->isa_pvectmask() &&
22415 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22416 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22417 format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
22418 ins_encode %{
22419 assert(UseAVX > 2, "required");
22420 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22421
22422 int vlen_enc = vector_length_encoding(this, $src1);
22423 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22424 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22425 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22426
22427 // Comparison i
22428 switch (src1_elem_bt) {
22429 case T_BYTE: {
22430 __ evpcmpb($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22431 break;
22432 }
22433 case T_SHORT: {
22434 __ evpcmpw($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22435 break;
22436 }
22437 case T_INT: {
22438 __ evpcmpd($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22439 break;
22440 }
22441 case T_LONG: {
22442 __ evpcmpq($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22443 break;
22444 }
22445 default: assert(false, "%s", type2name(src1_elem_bt));
22446 }
22447 %}
22448 ins_pipe( pipe_slow );
22449 %}
22450
22451 // Extract
22452
22453 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
22454 predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
22455 match(Set dst (ExtractI src idx));
22456 match(Set dst (ExtractS src idx));
22457 match(Set dst (ExtractB src idx));
22458 format %{ "extractI $dst,$src,$idx\t!" %}
22459 ins_encode %{
22460 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22461
22462 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22463 __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22464 %}
22465 ins_pipe( pipe_slow );
22466 %}
22467
22468 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
22469 predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
22470 Matcher::vector_length_in_bytes(n->in(1)) == 64); // src
22471 match(Set dst (ExtractI src idx));
22472 match(Set dst (ExtractS src idx));
22473 match(Set dst (ExtractB src idx));
22474 effect(TEMP vtmp);
22475 format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
22476 ins_encode %{
22477 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22478
22479 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22480 XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22481 __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
22482 %}
22483 ins_pipe( pipe_slow );
22484 %}
22485
22486 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
22487 predicate(Matcher::vector_length(n->in(1)) <= 2); // src
22488 match(Set dst (ExtractL src idx));
22489 format %{ "extractL $dst,$src,$idx\t!" %}
22490 ins_encode %{
22491 assert(UseSSE >= 4, "required");
22492 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22493
22494 __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22495 %}
22496 ins_pipe( pipe_slow );
22497 %}
22498
22499 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
22500 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22501 Matcher::vector_length(n->in(1)) == 8); // src
22502 match(Set dst (ExtractL src idx));
22503 effect(TEMP vtmp);
22504 format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
22505 ins_encode %{
22506 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22507
22508 XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22509 __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
22510 %}
22511 ins_pipe( pipe_slow );
22512 %}
22513
22514 instruct extractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22515 predicate(Matcher::vector_length(n->in(1)) <= 4);
22516 match(Set dst (ExtractF src idx));
22517 effect(TEMP dst, TEMP vtmp);
22518 format %{ "extractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22519 ins_encode %{
22520 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22521
22522 __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $vtmp$$XMMRegister);
22523 %}
22524 ins_pipe( pipe_slow );
22525 %}
22526
22527 instruct vextractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22528 predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
22529 Matcher::vector_length(n->in(1)/*src*/) == 16);
22530 match(Set dst (ExtractF src idx));
22531 effect(TEMP vtmp);
22532 format %{ "vextractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22533 ins_encode %{
22534 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22535
22536 XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22537 __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant);
22538 %}
22539 ins_pipe( pipe_slow );
22540 %}
22541
22542 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
22543 predicate(Matcher::vector_length(n->in(1)) == 2); // src
22544 match(Set dst (ExtractD src idx));
22545 format %{ "extractD $dst,$src,$idx\t!" %}
22546 ins_encode %{
22547 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22548
22549 __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22550 %}
22551 ins_pipe( pipe_slow );
22552 %}
22553
22554 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
22555 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22556 Matcher::vector_length(n->in(1)) == 8); // src
22557 match(Set dst (ExtractD src idx));
22558 effect(TEMP vtmp);
22559 format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
22560 ins_encode %{
22561 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22562
22563 XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22564 __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
22565 %}
22566 ins_pipe( pipe_slow );
22567 %}
22568
22569 // --------------------------------- Vector Blend --------------------------------------
22570
22571 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
22572 predicate(UseAVX == 0);
22573 match(Set dst (VectorBlend (Binary dst src) mask));
22574 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
22575 effect(TEMP tmp);
22576 ins_encode %{
22577 assert(UseSSE >= 4, "required");
22578
22579 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
22580 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
22581 }
22582 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
22583 %}
22584 ins_pipe( pipe_slow );
22585 %}
22586
22587 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
22588 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22589 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22590 Matcher::vector_length_in_bytes(n) <= 32 &&
22591 is_integral_type(Matcher::vector_element_basic_type(n)));
22592 match(Set dst (VectorBlend (Binary src1 src2) mask));
22593 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22594 ins_encode %{
22595 int vlen_enc = vector_length_encoding(this);
22596 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22597 %}
22598 ins_pipe( pipe_slow );
22599 %}
22600
22601 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
22602 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22603 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22604 Matcher::vector_length_in_bytes(n) <= 32 &&
22605 !is_integral_type(Matcher::vector_element_basic_type(n)));
22606 match(Set dst (VectorBlend (Binary src1 src2) mask));
22607 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22608 ins_encode %{
22609 int vlen_enc = vector_length_encoding(this);
22610 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22611 %}
22612 ins_pipe( pipe_slow );
22613 %}
22614
22615 instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
22616 predicate(UseAVX > 0 && EnableX86ECoreOpts &&
22617 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22618 Matcher::vector_length_in_bytes(n) <= 32);
22619 match(Set dst (VectorBlend (Binary src1 src2) mask));
22620 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
22621 effect(TEMP vtmp, TEMP dst);
22622 ins_encode %{
22623 int vlen_enc = vector_length_encoding(this);
22624 __ vpandn($vtmp$$XMMRegister, $mask$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22625 __ vpand ($dst$$XMMRegister, $mask$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22626 __ vpor ($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22627 %}
22628 ins_pipe( pipe_slow );
22629 %}
22630
22631 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
22632 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
22633 n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
22634 match(Set dst (VectorBlend (Binary src1 src2) mask));
22635 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22636 effect(TEMP ktmp);
22637 ins_encode %{
22638 int vlen_enc = Assembler::AVX_512bit;
22639 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22640 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
22641 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22642 %}
22643 ins_pipe( pipe_slow );
22644 %}
22645
22646
22647 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
22648 predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
22649 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
22650 VM_Version::supports_avx512bw()));
22651 match(Set dst (VectorBlend (Binary src1 src2) mask));
22652 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22653 ins_encode %{
22654 int vlen_enc = vector_length_encoding(this);
22655 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22656 __ evpblend(elem_bt, $dst$$XMMRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22657 %}
22658 ins_pipe( pipe_slow );
22659 %}
22660
22661 // --------------------------------- ABS --------------------------------------
22662 // a = |a|
22663 instruct vabsB_reg(vec dst, vec src) %{
22664 match(Set dst (AbsVB src));
22665 format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
22666 ins_encode %{
22667 uint vlen = Matcher::vector_length(this);
22668 if (vlen <= 16) {
22669 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22670 } else {
22671 int vlen_enc = vector_length_encoding(this);
22672 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22673 }
22674 %}
22675 ins_pipe( pipe_slow );
22676 %}
22677
22678 instruct vabsS_reg(vec dst, vec src) %{
22679 match(Set dst (AbsVS src));
22680 format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
22681 ins_encode %{
22682 uint vlen = Matcher::vector_length(this);
22683 if (vlen <= 8) {
22684 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22685 } else {
22686 int vlen_enc = vector_length_encoding(this);
22687 __ vpabsw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22688 }
22689 %}
22690 ins_pipe( pipe_slow );
22691 %}
22692
22693 instruct vabsI_reg(vec dst, vec src) %{
22694 match(Set dst (AbsVI src));
22695 format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
22696 ins_encode %{
22697 uint vlen = Matcher::vector_length(this);
22698 if (vlen <= 4) {
22699 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22700 } else {
22701 int vlen_enc = vector_length_encoding(this);
22702 __ vpabsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22703 }
22704 %}
22705 ins_pipe( pipe_slow );
22706 %}
22707
22708 instruct vabsL_reg(vec dst, vec src) %{
22709 match(Set dst (AbsVL src));
22710 format %{ "evpabsq $dst,$src\t# $dst = |$src| abs packedL" %}
22711 ins_encode %{
22712 assert(UseAVX > 2, "required");
22713 int vlen_enc = vector_length_encoding(this);
22714 if (!VM_Version::supports_avx512vl()) {
22715 vlen_enc = Assembler::AVX_512bit;
22716 }
22717 __ evpabsq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22718 %}
22719 ins_pipe( pipe_slow );
22720 %}
22721
22722 // --------------------------------- ABSNEG --------------------------------------
22723
22724 instruct vabsnegF(vec dst, vec src) %{
22725 predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
22726 match(Set dst (AbsVF src));
22727 match(Set dst (NegVF src));
22728 format %{ "vabsnegf $dst,$src,[mask]\t# absneg packedF" %}
22729 ins_cost(150);
22730 ins_encode %{
22731 int opcode = this->ideal_Opcode();
22732 int vlen = Matcher::vector_length(this);
22733 if (vlen == 2) {
22734 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22735 } else {
22736 assert(vlen == 8 || vlen == 16, "required");
22737 int vlen_enc = vector_length_encoding(this);
22738 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22739 }
22740 %}
22741 ins_pipe( pipe_slow );
22742 %}
22743
22744 instruct vabsneg4F(vec dst) %{
22745 predicate(Matcher::vector_length(n) == 4);
22746 match(Set dst (AbsVF dst));
22747 match(Set dst (NegVF dst));
22748 format %{ "vabsnegf $dst,[mask]\t# absneg packed4F" %}
22749 ins_cost(150);
22750 ins_encode %{
22751 int opcode = this->ideal_Opcode();
22752 __ vabsnegf(opcode, $dst$$XMMRegister, $dst$$XMMRegister);
22753 %}
22754 ins_pipe( pipe_slow );
22755 %}
22756
22757 instruct vabsnegD(vec dst, vec src) %{
22758 match(Set dst (AbsVD src));
22759 match(Set dst (NegVD src));
22760 format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
22761 ins_encode %{
22762 int opcode = this->ideal_Opcode();
22763 uint vlen = Matcher::vector_length(this);
22764 if (vlen == 2) {
22765 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22766 } else {
22767 int vlen_enc = vector_length_encoding(this);
22768 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22769 }
22770 %}
22771 ins_pipe( pipe_slow );
22772 %}
22773
22774 //------------------------------------- VectorTest --------------------------------------------
22775
22776 instruct vptest_lt16(rFlagsRegU cr, legVec src1, legVec src2, legVec vtmp) %{
22777 predicate(Matcher::vector_length_in_bytes(n->in(1)) < 16);
22778 match(Set cr (VectorTest src1 src2));
22779 effect(TEMP vtmp);
22780 format %{ "vptest_lt16 $src1, $src2\t! using $vtmp as TEMP" %}
22781 ins_encode %{
22782 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22783 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22784 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister, vlen);
22785 %}
22786 ins_pipe( pipe_slow );
22787 %}
22788
22789 instruct vptest_ge16(rFlagsRegU cr, legVec src1, legVec src2) %{
22790 predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16);
22791 match(Set cr (VectorTest src1 src2));
22792 format %{ "vptest_ge16 $src1, $src2\n\t" %}
22793 ins_encode %{
22794 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22795 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22796 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, vlen);
22797 %}
22798 ins_pipe( pipe_slow );
22799 %}
22800
22801 instruct ktest_alltrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22802 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22803 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22804 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
22805 match(Set cr (VectorTest src1 src2));
22806 effect(TEMP tmp);
22807 format %{ "ktest_alltrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22808 ins_encode %{
22809 uint masklen = Matcher::vector_length(this, $src1);
22810 __ kmovwl($tmp$$Register, $src1$$KRegister);
22811 __ andl($tmp$$Register, (1 << masklen) - 1);
22812 __ cmpl($tmp$$Register, (1 << masklen) - 1);
22813 %}
22814 ins_pipe( pipe_slow );
22815 %}
22816
22817 instruct ktest_anytrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22818 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22819 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22820 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
22821 match(Set cr (VectorTest src1 src2));
22822 effect(TEMP tmp);
22823 format %{ "ktest_anytrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22824 ins_encode %{
22825 uint masklen = Matcher::vector_length(this, $src1);
22826 __ kmovwl($tmp$$Register, $src1$$KRegister);
22827 __ andl($tmp$$Register, (1 << masklen) - 1);
22828 %}
22829 ins_pipe( pipe_slow );
22830 %}
22831
22832 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
22833 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
22834 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
22835 match(Set cr (VectorTest src1 src2));
22836 format %{ "ktest_ge8 $src1, $src2\n\t" %}
22837 ins_encode %{
22838 uint masklen = Matcher::vector_length(this, $src1);
22839 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
22840 %}
22841 ins_pipe( pipe_slow );
22842 %}
22843
22844 //------------------------------------- LoadMask --------------------------------------------
22845
22846 instruct loadMask(legVec dst, legVec src) %{
22847 predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
22848 match(Set dst (VectorLoadMask src));
22849 effect(TEMP dst);
22850 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
22851 ins_encode %{
22852 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22853 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22854 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
22855 %}
22856 ins_pipe( pipe_slow );
22857 %}
22858
22859 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
22860 predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22861 match(Set dst (VectorLoadMask src));
22862 effect(TEMP xtmp);
22863 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
22864 ins_encode %{
22865 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22866 true, Assembler::AVX_512bit);
22867 %}
22868 ins_pipe( pipe_slow );
22869 %}
22870
22871 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
22872 predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22873 match(Set dst (VectorLoadMask src));
22874 effect(TEMP xtmp);
22875 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
22876 ins_encode %{
22877 int vlen_enc = vector_length_encoding(in(1));
22878 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22879 false, vlen_enc);
22880 %}
22881 ins_pipe( pipe_slow );
22882 %}
22883
22884 //------------------------------------- StoreMask --------------------------------------------
22885
22886 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
22887 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22888 match(Set dst (VectorStoreMask src size));
22889 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22890 ins_encode %{
22891 int vlen = Matcher::vector_length(this);
22892 if (vlen <= 16 && UseAVX <= 2) {
22893 assert(UseSSE >= 3, "required");
22894 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22895 } else {
22896 assert(UseAVX > 0, "required");
22897 int src_vlen_enc = vector_length_encoding(this, $src);
22898 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22899 }
22900 %}
22901 ins_pipe( pipe_slow );
22902 %}
22903
22904 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
22905 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22906 match(Set dst (VectorStoreMask src size));
22907 effect(TEMP_DEF dst, TEMP xtmp);
22908 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22909 ins_encode %{
22910 int vlen_enc = Assembler::AVX_128bit;
22911 int vlen = Matcher::vector_length(this);
22912 if (vlen <= 8) {
22913 assert(UseSSE >= 3, "required");
22914 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22915 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22916 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22917 } else {
22918 assert(UseAVX > 0, "required");
22919 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22920 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22921 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22922 }
22923 %}
22924 ins_pipe( pipe_slow );
22925 %}
22926
22927 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
22928 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22929 match(Set dst (VectorStoreMask src size));
22930 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22931 effect(TEMP_DEF dst, TEMP xtmp);
22932 ins_encode %{
22933 int vlen_enc = Assembler::AVX_128bit;
22934 int vlen = Matcher::vector_length(this);
22935 if (vlen <= 4) {
22936 assert(UseSSE >= 3, "required");
22937 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22938 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22939 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22940 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22941 } else {
22942 assert(UseAVX > 0, "required");
22943 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22944 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22945 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22946 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22947 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22948 }
22949 %}
22950 ins_pipe( pipe_slow );
22951 %}
22952
22953 instruct storeMask8B(vec dst, vec src, vec xtmp, immI_8 size) %{
22954 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 2);
22955 match(Set dst (VectorStoreMask src size));
22956 effect(TEMP_DEF dst, TEMP xtmp);
22957 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22958 ins_encode %{
22959 assert(UseSSE >= 3, "required");
22960 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22961 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x8);
22962 __ pabsd($dst$$XMMRegister, $dst$$XMMRegister);
22963 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22964 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22965 %}
22966 ins_pipe( pipe_slow );
22967 %}
22968
22969 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
22970 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
22971 match(Set dst (VectorStoreMask src size));
22972 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
22973 effect(TEMP_DEF dst, TEMP vtmp);
22974 ins_encode %{
22975 int vlen_enc = Assembler::AVX_128bit;
22976 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
22977 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
22978 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
22979 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22980 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22981 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22982 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22983 %}
22984 ins_pipe( pipe_slow );
22985 %}
22986
22987 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
22988 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22989 match(Set dst (VectorStoreMask src size));
22990 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22991 ins_encode %{
22992 int src_vlen_enc = vector_length_encoding(this, $src);
22993 int dst_vlen_enc = vector_length_encoding(this);
22994 if (!VM_Version::supports_avx512vl()) {
22995 src_vlen_enc = Assembler::AVX_512bit;
22996 }
22997 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22998 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22999 %}
23000 ins_pipe( pipe_slow );
23001 %}
23002
23003 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
23004 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23005 match(Set dst (VectorStoreMask src size));
23006 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
23007 ins_encode %{
23008 int src_vlen_enc = vector_length_encoding(this, $src);
23009 int dst_vlen_enc = vector_length_encoding(this);
23010 if (!VM_Version::supports_avx512vl()) {
23011 src_vlen_enc = Assembler::AVX_512bit;
23012 }
23013 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
23014 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
23015 %}
23016 ins_pipe( pipe_slow );
23017 %}
23018
23019 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
23020 predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
23021 match(Set dst (VectorStoreMask mask size));
23022 effect(TEMP_DEF dst);
23023 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
23024 ins_encode %{
23025 assert(Matcher::vector_length_in_bytes(this, $mask) == 64, "");
23026 __ evmovdqul($dst$$XMMRegister, $mask$$KRegister, ExternalAddress(vector_int_mask_cmp_bits()),
23027 false, Assembler::AVX_512bit, noreg);
23028 __ evpmovdb($dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_512bit);
23029 %}
23030 ins_pipe( pipe_slow );
23031 %}
23032
23033 instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
23034 predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
23035 match(Set dst (VectorStoreMask mask size));
23036 effect(TEMP_DEF dst);
23037 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
23038 ins_encode %{
23039 int dst_vlen_enc = vector_length_encoding(this);
23040 __ evpmovm2b($dst$$XMMRegister, $mask$$KRegister, dst_vlen_enc);
23041 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
23042 %}
23043 ins_pipe( pipe_slow );
23044 %}
23045
23046 instruct vmaskcast_evex(kReg dst) %{
23047 match(Set dst (VectorMaskCast dst));
23048 ins_cost(0);
23049 format %{ "vector_mask_cast $dst" %}
23050 ins_encode %{
23051 // empty
23052 %}
23053 ins_pipe(empty);
23054 %}
23055
23056 instruct vmaskcast(vec dst) %{
23057 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
23058 match(Set dst (VectorMaskCast dst));
23059 ins_cost(0);
23060 format %{ "vector_mask_cast $dst" %}
23061 ins_encode %{
23062 // empty
23063 %}
23064 ins_pipe(empty);
23065 %}
23066
23067 instruct vmaskcast_avx(vec dst, vec src) %{
23068 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
23069 match(Set dst (VectorMaskCast src));
23070 format %{ "vector_mask_cast $dst, $src" %}
23071 ins_encode %{
23072 int vlen = Matcher::vector_length(this);
23073 BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
23074 BasicType dst_bt = Matcher::vector_element_basic_type(this);
23075 __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen);
23076 %}
23077 ins_pipe(pipe_slow);
23078 %}
23079
23080 //-------------------------------- Load Iota Indices ----------------------------------
23081
23082 instruct loadIotaIndices(vec dst, immI_0 src) %{
23083 match(Set dst (VectorLoadConst src));
23084 format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
23085 ins_encode %{
23086 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23087 BasicType bt = Matcher::vector_element_basic_type(this);
23088 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, bt);
23089 %}
23090 ins_pipe( pipe_slow );
23091 %}
23092
23093 instruct VectorPopulateIndex(vec dst, rRegI src1, immI_1 src2, vec vtmp) %{
23094 match(Set dst (PopulateIndex src1 src2));
23095 effect(TEMP dst, TEMP vtmp);
23096 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
23097 ins_encode %{
23098 assert($src2$$constant == 1, "required");
23099 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23100 int vlen_enc = vector_length_encoding(this);
23101 BasicType elem_bt = Matcher::vector_element_basic_type(this);
23102 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
23103 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
23104 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23105 %}
23106 ins_pipe( pipe_slow );
23107 %}
23108
23109 instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
23110 match(Set dst (PopulateIndex src1 src2));
23111 effect(TEMP dst, TEMP vtmp);
23112 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
23113 ins_encode %{
23114 assert($src2$$constant == 1, "required");
23115 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23116 int vlen_enc = vector_length_encoding(this);
23117 BasicType elem_bt = Matcher::vector_element_basic_type(this);
23118 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
23119 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
23120 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23121 %}
23122 ins_pipe( pipe_slow );
23123 %}
23124
23125 //-------------------------------- Rearrange ----------------------------------
23126
23127 // LoadShuffle/Rearrange for Byte
23128 instruct rearrangeB(vec dst, vec shuffle) %{
23129 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23130 Matcher::vector_length(n) < 32);
23131 match(Set dst (VectorRearrange dst shuffle));
23132 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23133 ins_encode %{
23134 assert(UseSSE >= 4, "required");
23135 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23136 %}
23137 ins_pipe( pipe_slow );
23138 %}
23139
23140 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
23141 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23142 Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
23143 match(Set dst (VectorRearrange src shuffle));
23144 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
23145 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
23146 ins_encode %{
23147 assert(UseAVX >= 2, "required");
23148 // Swap src into vtmp1
23149 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
23150 // Shuffle swapped src to get entries from other 128 bit lane
23151 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23152 // Shuffle original src to get entries from self 128 bit lane
23153 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23154 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
23155 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
23156 // Perform the blend
23157 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
23158 %}
23159 ins_pipe( pipe_slow );
23160 %}
23161
23162
23163 instruct rearrangeB_evex(vec dst, vec src, vec shuffle, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegI rtmp) %{
23164 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23165 Matcher::vector_length(n) > 32 && !VM_Version::supports_avx512_vbmi());
23166 match(Set dst (VectorRearrange src shuffle));
23167 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
23168 format %{ "vector_rearrange $dst, $shuffle, $src!\t using $xtmp1, $xtmp2, $xtmp3, $rtmp and $ktmp as TEMP" %}
23169 ins_encode %{
23170 int vlen_enc = vector_length_encoding(this);
23171 __ rearrange_bytes($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister,
23172 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister,
23173 $rtmp$$Register, $ktmp$$KRegister, vlen_enc);
23174 %}
23175 ins_pipe( pipe_slow );
23176 %}
23177
23178 instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
23179 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23180 Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
23181 match(Set dst (VectorRearrange src shuffle));
23182 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23183 ins_encode %{
23184 int vlen_enc = vector_length_encoding(this);
23185 __ vpermb($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23186 %}
23187 ins_pipe( pipe_slow );
23188 %}
23189
23190 // LoadShuffle/Rearrange for Short
23191
23192 instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
23193 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23194 !VM_Version::supports_avx512bw());
23195 match(Set dst (VectorLoadShuffle src));
23196 effect(TEMP dst, TEMP vtmp);
23197 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23198 ins_encode %{
23199 // Create a byte shuffle mask from short shuffle mask
23200 // only byte shuffle instruction available on these platforms
23201 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23202 if (UseAVX == 0) {
23203 assert(vlen_in_bytes <= 16, "required");
23204 // Multiply each shuffle by two to get byte index
23205 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
23206 __ psllw($vtmp$$XMMRegister, 1);
23207
23208 // Duplicate to create 2 copies of byte index
23209 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
23210 __ psllw($dst$$XMMRegister, 8);
23211 __ por($dst$$XMMRegister, $vtmp$$XMMRegister);
23212
23213 // Add one to get alternate byte index
23214 __ movdqu($vtmp$$XMMRegister, ExternalAddress(vector_short_shufflemask()), noreg);
23215 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
23216 } else {
23217 assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
23218 int vlen_enc = vector_length_encoding(this);
23219 // Multiply each shuffle by two to get byte index
23220 __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23221
23222 // Duplicate to create 2 copies of byte index
23223 __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
23224 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23225
23226 // Add one to get alternate byte index
23227 __ vpaddb($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_shufflemask()), vlen_enc, noreg);
23228 }
23229 %}
23230 ins_pipe( pipe_slow );
23231 %}
23232
23233 instruct rearrangeS(vec dst, vec shuffle) %{
23234 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23235 Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
23236 match(Set dst (VectorRearrange dst shuffle));
23237 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23238 ins_encode %{
23239 assert(UseSSE >= 4, "required");
23240 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23241 %}
23242 ins_pipe( pipe_slow );
23243 %}
23244
23245 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
23246 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23247 Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
23248 match(Set dst (VectorRearrange src shuffle));
23249 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
23250 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
23251 ins_encode %{
23252 assert(UseAVX >= 2, "required");
23253 // Swap src into vtmp1
23254 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
23255 // Shuffle swapped src to get entries from other 128 bit lane
23256 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23257 // Shuffle original src to get entries from self 128 bit lane
23258 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23259 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
23260 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
23261 // Perform the blend
23262 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
23263 %}
23264 ins_pipe( pipe_slow );
23265 %}
23266
23267 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
23268 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23269 VM_Version::supports_avx512bw());
23270 match(Set dst (VectorRearrange src shuffle));
23271 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23272 ins_encode %{
23273 int vlen_enc = vector_length_encoding(this);
23274 if (!VM_Version::supports_avx512vl()) {
23275 vlen_enc = Assembler::AVX_512bit;
23276 }
23277 __ vpermw($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23278 %}
23279 ins_pipe( pipe_slow );
23280 %}
23281
23282 // LoadShuffle/Rearrange for Integer and Float
23283
23284 instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
23285 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23286 Matcher::vector_length(n) == 4 && UseAVX == 0);
23287 match(Set dst (VectorLoadShuffle src));
23288 effect(TEMP dst, TEMP vtmp);
23289 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23290 ins_encode %{
23291 assert(UseSSE >= 4, "required");
23292
23293 // Create a byte shuffle mask from int shuffle mask
23294 // only byte shuffle instruction available on these platforms
23295
23296 // Duplicate and multiply each shuffle by 4
23297 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
23298 __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
23299 __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
23300 __ psllw($vtmp$$XMMRegister, 2);
23301
23302 // Duplicate again to create 4 copies of byte index
23303 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
23304 __ psllw($dst$$XMMRegister, 8);
23305 __ por($vtmp$$XMMRegister, $dst$$XMMRegister);
23306
23307 // Add 3,2,1,0 to get alternate byte index
23308 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_int_shufflemask()), noreg);
23309 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
23310 %}
23311 ins_pipe( pipe_slow );
23312 %}
23313
23314 instruct rearrangeI(vec dst, vec shuffle) %{
23315 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23316 UseAVX == 0);
23317 match(Set dst (VectorRearrange dst shuffle));
23318 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23319 ins_encode %{
23320 assert(UseSSE >= 4, "required");
23321 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23322 %}
23323 ins_pipe( pipe_slow );
23324 %}
23325
23326 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
23327 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23328 UseAVX > 0);
23329 match(Set dst (VectorRearrange src shuffle));
23330 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23331 ins_encode %{
23332 int vlen_enc = vector_length_encoding(this);
23333 BasicType bt = Matcher::vector_element_basic_type(this);
23334 __ vector_rearrange_int_float(bt, $dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23335 %}
23336 ins_pipe( pipe_slow );
23337 %}
23338
23339 // LoadShuffle/Rearrange for Long and Double
23340
23341 instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
23342 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23343 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23344 match(Set dst (VectorLoadShuffle src));
23345 effect(TEMP dst, TEMP vtmp);
23346 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23347 ins_encode %{
23348 assert(UseAVX >= 2, "required");
23349
23350 int vlen_enc = vector_length_encoding(this);
23351 // Create a double word shuffle mask from long shuffle mask
23352 // only double word shuffle instruction available on these platforms
23353
23354 // Multiply each shuffle by two to get double word index
23355 __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23356
23357 // Duplicate each double word shuffle
23358 __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
23359 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23360
23361 // Add one to get alternate double word index
23362 __ vpaddd($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_long_shufflemask()), vlen_enc, noreg);
23363 %}
23364 ins_pipe( pipe_slow );
23365 %}
23366
23367 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
23368 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23369 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23370 match(Set dst (VectorRearrange src shuffle));
23371 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23372 ins_encode %{
23373 assert(UseAVX >= 2, "required");
23374
23375 int vlen_enc = vector_length_encoding(this);
23376 __ vpermd($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23377 %}
23378 ins_pipe( pipe_slow );
23379 %}
23380
23381 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
23382 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23383 (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
23384 match(Set dst (VectorRearrange src shuffle));
23385 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23386 ins_encode %{
23387 assert(UseAVX > 2, "required");
23388
23389 int vlen_enc = vector_length_encoding(this);
23390 if (vlen_enc == Assembler::AVX_128bit) {
23391 vlen_enc = Assembler::AVX_256bit;
23392 }
23393 __ vpermq($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23394 %}
23395 ins_pipe( pipe_slow );
23396 %}
23397
23398 // --------------------------------- FMA --------------------------------------
23399 // a * b + c
23400
23401 instruct vfmaF_reg(vec a, vec b, vec c) %{
23402 match(Set c (FmaVF c (Binary a b)));
23403 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23404 ins_cost(150);
23405 ins_encode %{
23406 assert(UseFMA, "not enabled");
23407 int vlen_enc = vector_length_encoding(this);
23408 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23409 %}
23410 ins_pipe( pipe_slow );
23411 %}
23412
23413 instruct vfmaF_mem(vec a, memory b, vec c) %{
23414 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23415 match(Set c (FmaVF c (Binary a (LoadVector b))));
23416 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23417 ins_cost(150);
23418 ins_encode %{
23419 assert(UseFMA, "not enabled");
23420 int vlen_enc = vector_length_encoding(this);
23421 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23422 %}
23423 ins_pipe( pipe_slow );
23424 %}
23425
23426 instruct vfmaD_reg(vec a, vec b, vec c) %{
23427 match(Set c (FmaVD c (Binary a b)));
23428 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23429 ins_cost(150);
23430 ins_encode %{
23431 assert(UseFMA, "not enabled");
23432 int vlen_enc = vector_length_encoding(this);
23433 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23434 %}
23435 ins_pipe( pipe_slow );
23436 %}
23437
23438 instruct vfmaD_mem(vec a, memory b, vec c) %{
23439 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23440 match(Set c (FmaVD c (Binary a (LoadVector b))));
23441 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23442 ins_cost(150);
23443 ins_encode %{
23444 assert(UseFMA, "not enabled");
23445 int vlen_enc = vector_length_encoding(this);
23446 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23447 %}
23448 ins_pipe( pipe_slow );
23449 %}
23450
23451 // --------------------------------- Vector Multiply Add --------------------------------------
23452
23453 instruct vmuladdS2I_reg_sse(vec dst, vec src1) %{
23454 predicate(UseAVX == 0);
23455 match(Set dst (MulAddVS2VI dst src1));
23456 format %{ "pmaddwd $dst,$src1\t! muladd packedStoI" %}
23457 ins_encode %{
23458 __ pmaddwd($dst$$XMMRegister, $src1$$XMMRegister);
23459 %}
23460 ins_pipe( pipe_slow );
23461 %}
23462
23463 instruct vmuladdS2I_reg_avx(vec dst, vec src1, vec src2) %{
23464 predicate(UseAVX > 0);
23465 match(Set dst (MulAddVS2VI src1 src2));
23466 format %{ "vpmaddwd $dst,$src1,$src2\t! muladd packedStoI" %}
23467 ins_encode %{
23468 int vlen_enc = vector_length_encoding(this);
23469 __ vpmaddwd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23470 %}
23471 ins_pipe( pipe_slow );
23472 %}
23473
23474 // --------------------------------- Vector Multiply Add Add ----------------------------------
23475
23476 instruct vmuladdaddS2I_reg(vec dst, vec src1, vec src2) %{
23477 predicate(VM_Version::supports_avx512_vnni());
23478 match(Set dst (AddVI (MulAddVS2VI src1 src2) dst));
23479 format %{ "evpdpwssd $dst,$src1,$src2\t! muladdadd packedStoI" %}
23480 ins_encode %{
23481 assert(UseAVX > 2, "required");
23482 int vlen_enc = vector_length_encoding(this);
23483 __ evpdpwssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23484 %}
23485 ins_pipe( pipe_slow );
23486 ins_cost(10);
23487 %}
23488
23489 // --------------------------------- PopCount --------------------------------------
23490
23491 instruct vpopcount_integral_reg_evex(vec dst, vec src) %{
23492 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23493 match(Set dst (PopCountVI src));
23494 match(Set dst (PopCountVL src));
23495 format %{ "vector_popcount_integral $dst, $src" %}
23496 ins_encode %{
23497 int opcode = this->ideal_Opcode();
23498 int vlen_enc = vector_length_encoding(this, $src);
23499 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23500 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, k0, true, vlen_enc);
23501 %}
23502 ins_pipe( pipe_slow );
23503 %}
23504
23505 instruct vpopcount_integral_reg_evex_masked(vec dst, vec src, kReg mask) %{
23506 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23507 match(Set dst (PopCountVI src mask));
23508 match(Set dst (PopCountVL src mask));
23509 format %{ "vector_popcount_integral_masked $dst, $src, $mask" %}
23510 ins_encode %{
23511 int vlen_enc = vector_length_encoding(this, $src);
23512 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23513 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23514 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, true, vlen_enc);
23515 %}
23516 ins_pipe( pipe_slow );
23517 %}
23518
23519 instruct vpopcount_avx_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegP rtmp) %{
23520 predicate(!is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23521 match(Set dst (PopCountVI src));
23522 match(Set dst (PopCountVL src));
23523 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23524 format %{ "vector_popcount_integral $dst, $src\t! using $xtmp1, $xtmp2, and $rtmp as TEMP" %}
23525 ins_encode %{
23526 int opcode = this->ideal_Opcode();
23527 int vlen_enc = vector_length_encoding(this, $src);
23528 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23529 __ vector_popcount_integral(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23530 $xtmp2$$XMMRegister, $rtmp$$Register, vlen_enc);
23531 %}
23532 ins_pipe( pipe_slow );
23533 %}
23534
23535 // --------------------------------- Vector Trailing Zeros Count --------------------------------------
23536
23537 instruct vcount_trailing_zeros_reg_evex(vec dst, vec src, vec xtmp, rRegP rtmp) %{
23538 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23539 Matcher::vector_length_in_bytes(n->in(1))));
23540 match(Set dst (CountTrailingZerosV src));
23541 effect(TEMP dst, TEMP xtmp, TEMP rtmp);
23542 ins_cost(400);
23543 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp and $rtmp as TEMP" %}
23544 ins_encode %{
23545 int vlen_enc = vector_length_encoding(this, $src);
23546 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23547 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23548 xnoreg, xnoreg, $xtmp$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23549 %}
23550 ins_pipe( pipe_slow );
23551 %}
23552
23553 instruct vcount_trailing_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23554 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23555 VM_Version::supports_avx512cd() &&
23556 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23557 match(Set dst (CountTrailingZerosV src));
23558 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23559 ins_cost(400);
23560 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3 and $rtmp as TEMP" %}
23561 ins_encode %{
23562 int vlen_enc = vector_length_encoding(this, $src);
23563 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23564 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23565 $xtmp2$$XMMRegister, xnoreg, $xtmp3$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23566 %}
23567 ins_pipe( pipe_slow );
23568 %}
23569
23570 instruct vcount_trailing_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, kReg ktmp, rRegP rtmp) %{
23571 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23572 match(Set dst (CountTrailingZerosV src));
23573 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP ktmp, TEMP rtmp);
23574 ins_cost(400);
23575 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $xtmp4, $ktmp and $rtmp as TEMP" %}
23576 ins_encode %{
23577 int vlen_enc = vector_length_encoding(this, $src);
23578 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23579 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23580 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
23581 $ktmp$$KRegister, $rtmp$$Register, vlen_enc);
23582 %}
23583 ins_pipe( pipe_slow );
23584 %}
23585
23586 instruct vcount_trailing_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23587 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23588 match(Set dst (CountTrailingZerosV src));
23589 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23590 format %{ "vector_count_trailing_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23591 ins_encode %{
23592 int vlen_enc = vector_length_encoding(this, $src);
23593 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23594 __ vector_count_trailing_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23595 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23596 %}
23597 ins_pipe( pipe_slow );
23598 %}
23599
23600
23601 // --------------------------------- Bitwise Ternary Logic ----------------------------------
23602
23603 instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
23604 match(Set dst (MacroLogicV (Binary dst src2) (Binary src3 func)));
23605 effect(TEMP dst);
23606 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23607 ins_encode %{
23608 int vector_len = vector_length_encoding(this);
23609 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$XMMRegister, vector_len);
23610 %}
23611 ins_pipe( pipe_slow );
23612 %}
23613
23614 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
23615 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
23616 match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
23617 effect(TEMP dst);
23618 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23619 ins_encode %{
23620 int vector_len = vector_length_encoding(this);
23621 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$Address, vector_len);
23622 %}
23623 ins_pipe( pipe_slow );
23624 %}
23625
23626 // --------------------------------- Rotation Operations ----------------------------------
23627 instruct vprotate_immI8(vec dst, vec src, immI8 shift) %{
23628 match(Set dst (RotateLeftV src shift));
23629 match(Set dst (RotateRightV src shift));
23630 format %{ "vprotate_imm8 $dst,$src,$shift\t! vector rotate" %}
23631 ins_encode %{
23632 int opcode = this->ideal_Opcode();
23633 int vector_len = vector_length_encoding(this);
23634 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23635 __ vprotate_imm(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
23636 %}
23637 ins_pipe( pipe_slow );
23638 %}
23639
23640 instruct vprorate(vec dst, vec src, vec shift) %{
23641 match(Set dst (RotateLeftV src shift));
23642 match(Set dst (RotateRightV src shift));
23643 format %{ "vprotate $dst,$src,$shift\t! vector rotate" %}
23644 ins_encode %{
23645 int opcode = this->ideal_Opcode();
23646 int vector_len = vector_length_encoding(this);
23647 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23648 __ vprotate_var(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
23649 %}
23650 ins_pipe( pipe_slow );
23651 %}
23652
23653 // ---------------------------------- Masked Operations ------------------------------------
23654 instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
23655 predicate(!n->in(3)->bottom_type()->isa_pvectmask());
23656 match(Set dst (LoadVectorMasked mem mask));
23657 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23658 ins_encode %{
23659 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23660 int vlen_enc = vector_length_encoding(this);
23661 __ vmovmask(elmType, $dst$$XMMRegister, $mem$$Address, $mask$$XMMRegister, vlen_enc);
23662 %}
23663 ins_pipe( pipe_slow );
23664 %}
23665
23666
23667 instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
23668 predicate(n->in(3)->bottom_type()->isa_pvectmask());
23669 match(Set dst (LoadVectorMasked mem mask));
23670 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23671 ins_encode %{
23672 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23673 int vector_len = vector_length_encoding(this);
23674 __ evmovdqu(elmType, $mask$$KRegister, $dst$$XMMRegister, $mem$$Address, false, vector_len);
23675 %}
23676 ins_pipe( pipe_slow );
23677 %}
23678
23679 instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
23680 predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23681 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23682 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23683 ins_encode %{
23684 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23685 int vlen_enc = vector_length_encoding(src_node);
23686 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23687 __ vmovmask(elmType, $mem$$Address, $src$$XMMRegister, $mask$$XMMRegister, vlen_enc);
23688 %}
23689 ins_pipe( pipe_slow );
23690 %}
23691
23692 instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
23693 predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23694 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23695 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23696 ins_encode %{
23697 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23698 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23699 int vlen_enc = vector_length_encoding(src_node);
23700 __ evmovdqu(elmType, $mask$$KRegister, $mem$$Address, $src$$XMMRegister, true, vlen_enc);
23701 %}
23702 ins_pipe( pipe_slow );
23703 %}
23704
23705 instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
23706 match(Set addr (VerifyVectorAlignment addr mask));
23707 effect(KILL cr);
23708 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
23709 ins_encode %{
23710 Label Lskip;
23711 // check if masked bits of addr are zero
23712 __ testq($addr$$Register, $mask$$constant);
23713 __ jccb(Assembler::equal, Lskip);
23714 __ stop("verify_vector_alignment found a misaligned vector memory access");
23715 __ bind(Lskip);
23716 %}
23717 ins_pipe(pipe_slow);
23718 %}
23719
23720 instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
23721 match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
23722 effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
23723 format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
23724 ins_encode %{
23725 assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
23726 assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
23727
23728 Label DONE;
23729 int vlen_enc = vector_length_encoding(this, $src1);
23730 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
23731
23732 __ knotql($ktmp2$$KRegister, $mask$$KRegister);
23733 __ mov64($dst$$Register, -1L);
23734 __ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
23735 __ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
23736 __ jccb(Assembler::carrySet, DONE);
23737 __ kmovql($dst$$Register, $ktmp1$$KRegister);
23738 __ notq($dst$$Register);
23739 __ tzcntq($dst$$Register, $dst$$Register);
23740 __ bind(DONE);
23741 %}
23742 ins_pipe( pipe_slow );
23743 %}
23744
23745
23746 instruct vmask_gen(kReg dst, rRegL len, rRegL temp, rFlagsReg cr) %{
23747 match(Set dst (VectorMaskGen len));
23748 effect(TEMP temp, KILL cr);
23749 format %{ "vector_mask_gen32 $dst, $len \t! vector mask generator" %}
23750 ins_encode %{
23751 __ genmask($dst$$KRegister, $len$$Register, $temp$$Register);
23752 %}
23753 ins_pipe( pipe_slow );
23754 %}
23755
23756 instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
23757 match(Set dst (VectorMaskGen len));
23758 format %{ "vector_mask_gen $len \t! vector mask generator" %}
23759 effect(TEMP temp);
23760 ins_encode %{
23761 if ($len$$constant > 0) {
23762 __ mov64($temp$$Register, right_n_bits($len$$constant));
23763 __ kmovql($dst$$KRegister, $temp$$Register);
23764 } else {
23765 __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
23766 }
23767 %}
23768 ins_pipe( pipe_slow );
23769 %}
23770
23771 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
23772 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23773 match(Set dst (VectorMaskToLong mask));
23774 effect(TEMP dst, KILL cr);
23775 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
23776 ins_encode %{
23777 int opcode = this->ideal_Opcode();
23778 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23779 int mask_len = Matcher::vector_length(this, $mask);
23780 int mask_size = mask_len * type2aelembytes(mbt);
23781 int vlen_enc = vector_length_encoding(this, $mask);
23782 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23783 $dst$$Register, mask_len, mask_size, vlen_enc);
23784 %}
23785 ins_pipe( pipe_slow );
23786 %}
23787
23788 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
23789 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23790 match(Set dst (VectorMaskToLong mask));
23791 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
23792 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23793 ins_encode %{
23794 int opcode = this->ideal_Opcode();
23795 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23796 int mask_len = Matcher::vector_length(this, $mask);
23797 int vlen_enc = vector_length_encoding(this, $mask);
23798 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23799 $dst$$Register, mask_len, mbt, vlen_enc);
23800 %}
23801 ins_pipe( pipe_slow );
23802 %}
23803
23804 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
23805 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23806 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
23807 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
23808 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23809 ins_encode %{
23810 int opcode = this->ideal_Opcode();
23811 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23812 int mask_len = Matcher::vector_length(this, $mask);
23813 int vlen_enc = vector_length_encoding(this, $mask);
23814 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23815 $dst$$Register, mask_len, mbt, vlen_enc);
23816 %}
23817 ins_pipe( pipe_slow );
23818 %}
23819
23820 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23821 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23822 match(Set dst (VectorMaskTrueCount mask));
23823 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23824 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
23825 ins_encode %{
23826 int opcode = this->ideal_Opcode();
23827 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23828 int mask_len = Matcher::vector_length(this, $mask);
23829 int mask_size = mask_len * type2aelembytes(mbt);
23830 int vlen_enc = vector_length_encoding(this, $mask);
23831 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23832 $tmp$$Register, mask_len, mask_size, vlen_enc);
23833 %}
23834 ins_pipe( pipe_slow );
23835 %}
23836
23837 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23838 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23839 match(Set dst (VectorMaskTrueCount mask));
23840 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23841 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23842 ins_encode %{
23843 int opcode = this->ideal_Opcode();
23844 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23845 int mask_len = Matcher::vector_length(this, $mask);
23846 int vlen_enc = vector_length_encoding(this, $mask);
23847 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23848 $tmp$$Register, mask_len, mbt, vlen_enc);
23849 %}
23850 ins_pipe( pipe_slow );
23851 %}
23852
23853 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23854 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23855 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
23856 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23857 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23858 ins_encode %{
23859 int opcode = this->ideal_Opcode();
23860 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23861 int mask_len = Matcher::vector_length(this, $mask);
23862 int vlen_enc = vector_length_encoding(this, $mask);
23863 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23864 $tmp$$Register, mask_len, mbt, vlen_enc);
23865 %}
23866 ins_pipe( pipe_slow );
23867 %}
23868
23869 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23870 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23871 match(Set dst (VectorMaskFirstTrue mask));
23872 match(Set dst (VectorMaskLastTrue mask));
23873 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23874 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
23875 ins_encode %{
23876 int opcode = this->ideal_Opcode();
23877 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23878 int mask_len = Matcher::vector_length(this, $mask);
23879 int mask_size = mask_len * type2aelembytes(mbt);
23880 int vlen_enc = vector_length_encoding(this, $mask);
23881 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23882 $tmp$$Register, mask_len, mask_size, vlen_enc);
23883 %}
23884 ins_pipe( pipe_slow );
23885 %}
23886
23887 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23888 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23889 match(Set dst (VectorMaskFirstTrue mask));
23890 match(Set dst (VectorMaskLastTrue mask));
23891 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23892 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23893 ins_encode %{
23894 int opcode = this->ideal_Opcode();
23895 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23896 int mask_len = Matcher::vector_length(this, $mask);
23897 int vlen_enc = vector_length_encoding(this, $mask);
23898 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23899 $tmp$$Register, mask_len, mbt, vlen_enc);
23900 %}
23901 ins_pipe( pipe_slow );
23902 %}
23903
23904 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23905 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23906 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
23907 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
23908 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23909 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23910 ins_encode %{
23911 int opcode = this->ideal_Opcode();
23912 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23913 int mask_len = Matcher::vector_length(this, $mask);
23914 int vlen_enc = vector_length_encoding(this, $mask);
23915 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23916 $tmp$$Register, mask_len, mbt, vlen_enc);
23917 %}
23918 ins_pipe( pipe_slow );
23919 %}
23920
23921 // --------------------------------- Compress/Expand Operations ---------------------------
23922 instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
23923 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
23924 match(Set dst (CompressV src mask));
23925 match(Set dst (ExpandV src mask));
23926 effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
23927 format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
23928 ins_encode %{
23929 int opcode = this->ideal_Opcode();
23930 int vlen_enc = vector_length_encoding(this);
23931 BasicType bt = Matcher::vector_element_basic_type(this);
23932 __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
23933 $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
23934 %}
23935 ins_pipe( pipe_slow );
23936 %}
23937
23938 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
23939 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
23940 match(Set dst (CompressV src mask));
23941 match(Set dst (ExpandV src mask));
23942 format %{ "vector_compress_expand $dst, $src, $mask" %}
23943 ins_encode %{
23944 int opcode = this->ideal_Opcode();
23945 int vector_len = vector_length_encoding(this);
23946 BasicType bt = Matcher::vector_element_basic_type(this);
23947 __ vector_compress_expand(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, false, bt, vector_len);
23948 %}
23949 ins_pipe( pipe_slow );
23950 %}
23951
23952 instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
23953 match(Set dst (CompressM mask));
23954 effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
23955 format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
23956 ins_encode %{
23957 assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
23958 int mask_len = Matcher::vector_length(this);
23959 __ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
23960 %}
23961 ins_pipe( pipe_slow );
23962 %}
23963
23964 // -------------------------------- Bit and Byte Reversal Vector Operations ------------------------
23965
23966 instruct vreverse_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23967 predicate(!VM_Version::supports_gfni());
23968 match(Set dst (ReverseV src));
23969 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23970 format %{ "vector_reverse_bit_evex $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23971 ins_encode %{
23972 int vec_enc = vector_length_encoding(this);
23973 BasicType bt = Matcher::vector_element_basic_type(this);
23974 __ vector_reverse_bit(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23975 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23976 %}
23977 ins_pipe( pipe_slow );
23978 %}
23979
23980 instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
23981 predicate(VM_Version::supports_gfni());
23982 match(Set dst (ReverseV src));
23983 effect(TEMP dst, TEMP xtmp);
23984 format %{ "vector_reverse_bit_gfni $dst, $src!\t using $xtmp as TEMP" %}
23985 ins_encode %{
23986 int vec_enc = vector_length_encoding(this);
23987 BasicType bt = Matcher::vector_element_basic_type(this);
23988 InternalAddress addr = $constantaddress(jlong(0x8040201008040201));
23989 __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
23990 $xtmp$$XMMRegister);
23991 %}
23992 ins_pipe( pipe_slow );
23993 %}
23994
23995 instruct vreverse_byte_reg(vec dst, vec src) %{
23996 predicate(VM_Version::supports_avx512bw() || Matcher::vector_length_in_bytes(n) < 64);
23997 match(Set dst (ReverseBytesV src));
23998 effect(TEMP dst);
23999 format %{ "vector_reverse_byte $dst, $src" %}
24000 ins_encode %{
24001 int vec_enc = vector_length_encoding(this);
24002 BasicType bt = Matcher::vector_element_basic_type(this);
24003 __ vector_reverse_byte(bt, $dst$$XMMRegister, $src$$XMMRegister, vec_enc);
24004 %}
24005 ins_pipe( pipe_slow );
24006 %}
24007
24008 instruct vreverse_byte64_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
24009 predicate(!VM_Version::supports_avx512bw() && Matcher::vector_length_in_bytes(n) == 64);
24010 match(Set dst (ReverseBytesV src));
24011 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
24012 format %{ "vector_reverse_byte $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
24013 ins_encode %{
24014 int vec_enc = vector_length_encoding(this);
24015 BasicType bt = Matcher::vector_element_basic_type(this);
24016 __ vector_reverse_byte64(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24017 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
24018 %}
24019 ins_pipe( pipe_slow );
24020 %}
24021
24022 // ---------------------------------- Vector Count Leading Zeros -----------------------------------
24023
24024 instruct vcount_leading_zeros_IL_reg_evex(vec dst, vec src) %{
24025 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
24026 Matcher::vector_length_in_bytes(n->in(1))));
24027 match(Set dst (CountLeadingZerosV src));
24028 format %{ "vector_count_leading_zeros $dst, $src" %}
24029 ins_encode %{
24030 int vlen_enc = vector_length_encoding(this, $src);
24031 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24032 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
24033 xnoreg, xnoreg, k0, noreg, true, vlen_enc);
24034 %}
24035 ins_pipe( pipe_slow );
24036 %}
24037
24038 instruct vcount_leading_zeros_IL_reg_evex_masked(vec dst, vec src, kReg mask) %{
24039 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
24040 Matcher::vector_length_in_bytes(n->in(1))));
24041 match(Set dst (CountLeadingZerosV src mask));
24042 format %{ "vector_count_leading_zeros $dst, $src, $mask" %}
24043 ins_encode %{
24044 int vlen_enc = vector_length_encoding(this, $src);
24045 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24046 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
24047 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg,
24048 xnoreg, $mask$$KRegister, noreg, true, vlen_enc);
24049 %}
24050 ins_pipe( pipe_slow );
24051 %}
24052
24053 instruct vcount_leading_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2) %{
24054 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
24055 VM_Version::supports_avx512cd() &&
24056 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
24057 match(Set dst (CountLeadingZerosV src));
24058 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24059 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1 and $xtmp2 as TEMP" %}
24060 ins_encode %{
24061 int vlen_enc = vector_length_encoding(this, $src);
24062 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24063 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24064 $xtmp2$$XMMRegister, xnoreg, k0, noreg, true, vlen_enc);
24065 %}
24066 ins_pipe( pipe_slow );
24067 %}
24068
24069 instruct vcount_leading_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegP rtmp) %{
24070 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
24071 match(Set dst (CountLeadingZerosV src));
24072 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
24073 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $ktmp and $rtmp as TEMP" %}
24074 ins_encode %{
24075 int vlen_enc = vector_length_encoding(this, $src);
24076 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24077 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24078 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $ktmp$$KRegister,
24079 $rtmp$$Register, true, vlen_enc);
24080 %}
24081 ins_pipe( pipe_slow );
24082 %}
24083
24084 instruct vcount_leading_zeros_int_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3) %{
24085 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_INT &&
24086 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
24087 match(Set dst (CountLeadingZerosV src));
24088 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24089 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24090 ins_encode %{
24091 int vlen_enc = vector_length_encoding(this, $src);
24092 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24093 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24094 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, noreg, vlen_enc);
24095 %}
24096 ins_pipe( pipe_slow );
24097 %}
24098
24099 instruct vcount_leading_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
24100 predicate(Matcher::vector_element_basic_type(n->in(1)) != T_INT &&
24101 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
24102 match(Set dst (CountLeadingZerosV src));
24103 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
24104 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
24105 ins_encode %{
24106 int vlen_enc = vector_length_encoding(this, $src);
24107 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24108 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24109 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
24110 %}
24111 ins_pipe( pipe_slow );
24112 %}
24113
24114 // ---------------------------------- Vector Masked Operations ------------------------------------
24115
24116 instruct vadd_reg_masked(vec dst, vec src2, kReg mask) %{
24117 match(Set dst (AddVB (Binary dst src2) mask));
24118 match(Set dst (AddVS (Binary dst src2) mask));
24119 match(Set dst (AddVI (Binary dst src2) mask));
24120 match(Set dst (AddVL (Binary dst src2) mask));
24121 match(Set dst (AddVF (Binary dst src2) mask));
24122 match(Set dst (AddVD (Binary dst src2) mask));
24123 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
24124 ins_encode %{
24125 int vlen_enc = vector_length_encoding(this);
24126 BasicType bt = Matcher::vector_element_basic_type(this);
24127 int opc = this->ideal_Opcode();
24128 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24129 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24130 %}
24131 ins_pipe( pipe_slow );
24132 %}
24133
24134 instruct vadd_mem_masked(vec dst, memory src2, kReg mask) %{
24135 match(Set dst (AddVB (Binary dst (LoadVector src2)) mask));
24136 match(Set dst (AddVS (Binary dst (LoadVector src2)) mask));
24137 match(Set dst (AddVI (Binary dst (LoadVector src2)) mask));
24138 match(Set dst (AddVL (Binary dst (LoadVector src2)) mask));
24139 match(Set dst (AddVF (Binary dst (LoadVector src2)) mask));
24140 match(Set dst (AddVD (Binary dst (LoadVector src2)) mask));
24141 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
24142 ins_encode %{
24143 int vlen_enc = vector_length_encoding(this);
24144 BasicType bt = Matcher::vector_element_basic_type(this);
24145 int opc = this->ideal_Opcode();
24146 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24147 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24148 %}
24149 ins_pipe( pipe_slow );
24150 %}
24151
24152 instruct vxor_reg_masked(vec dst, vec src2, kReg mask) %{
24153 match(Set dst (XorV (Binary dst src2) mask));
24154 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
24155 ins_encode %{
24156 int vlen_enc = vector_length_encoding(this);
24157 BasicType bt = Matcher::vector_element_basic_type(this);
24158 int opc = this->ideal_Opcode();
24159 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24160 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24161 %}
24162 ins_pipe( pipe_slow );
24163 %}
24164
24165 instruct vxor_mem_masked(vec dst, memory src2, kReg mask) %{
24166 match(Set dst (XorV (Binary dst (LoadVector src2)) mask));
24167 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
24168 ins_encode %{
24169 int vlen_enc = vector_length_encoding(this);
24170 BasicType bt = Matcher::vector_element_basic_type(this);
24171 int opc = this->ideal_Opcode();
24172 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24173 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24174 %}
24175 ins_pipe( pipe_slow );
24176 %}
24177
24178 instruct vor_reg_masked(vec dst, vec src2, kReg mask) %{
24179 match(Set dst (OrV (Binary dst src2) mask));
24180 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
24181 ins_encode %{
24182 int vlen_enc = vector_length_encoding(this);
24183 BasicType bt = Matcher::vector_element_basic_type(this);
24184 int opc = this->ideal_Opcode();
24185 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24186 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24187 %}
24188 ins_pipe( pipe_slow );
24189 %}
24190
24191 instruct vor_mem_masked(vec dst, memory src2, kReg mask) %{
24192 match(Set dst (OrV (Binary dst (LoadVector src2)) mask));
24193 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
24194 ins_encode %{
24195 int vlen_enc = vector_length_encoding(this);
24196 BasicType bt = Matcher::vector_element_basic_type(this);
24197 int opc = this->ideal_Opcode();
24198 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24199 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24200 %}
24201 ins_pipe( pipe_slow );
24202 %}
24203
24204 instruct vand_reg_masked(vec dst, vec src2, kReg mask) %{
24205 match(Set dst (AndV (Binary dst src2) mask));
24206 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
24207 ins_encode %{
24208 int vlen_enc = vector_length_encoding(this);
24209 BasicType bt = Matcher::vector_element_basic_type(this);
24210 int opc = this->ideal_Opcode();
24211 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24212 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24213 %}
24214 ins_pipe( pipe_slow );
24215 %}
24216
24217 instruct vand_mem_masked(vec dst, memory src2, kReg mask) %{
24218 match(Set dst (AndV (Binary dst (LoadVector src2)) mask));
24219 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
24220 ins_encode %{
24221 int vlen_enc = vector_length_encoding(this);
24222 BasicType bt = Matcher::vector_element_basic_type(this);
24223 int opc = this->ideal_Opcode();
24224 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24225 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24226 %}
24227 ins_pipe( pipe_slow );
24228 %}
24229
24230 instruct vsub_reg_masked(vec dst, vec src2, kReg mask) %{
24231 match(Set dst (SubVB (Binary dst src2) mask));
24232 match(Set dst (SubVS (Binary dst src2) mask));
24233 match(Set dst (SubVI (Binary dst src2) mask));
24234 match(Set dst (SubVL (Binary dst src2) mask));
24235 match(Set dst (SubVF (Binary dst src2) mask));
24236 match(Set dst (SubVD (Binary dst src2) mask));
24237 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
24238 ins_encode %{
24239 int vlen_enc = vector_length_encoding(this);
24240 BasicType bt = Matcher::vector_element_basic_type(this);
24241 int opc = this->ideal_Opcode();
24242 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24243 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24244 %}
24245 ins_pipe( pipe_slow );
24246 %}
24247
24248 instruct vsub_mem_masked(vec dst, memory src2, kReg mask) %{
24249 match(Set dst (SubVB (Binary dst (LoadVector src2)) mask));
24250 match(Set dst (SubVS (Binary dst (LoadVector src2)) mask));
24251 match(Set dst (SubVI (Binary dst (LoadVector src2)) mask));
24252 match(Set dst (SubVL (Binary dst (LoadVector src2)) mask));
24253 match(Set dst (SubVF (Binary dst (LoadVector src2)) mask));
24254 match(Set dst (SubVD (Binary dst (LoadVector src2)) mask));
24255 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
24256 ins_encode %{
24257 int vlen_enc = vector_length_encoding(this);
24258 BasicType bt = Matcher::vector_element_basic_type(this);
24259 int opc = this->ideal_Opcode();
24260 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24261 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24262 %}
24263 ins_pipe( pipe_slow );
24264 %}
24265
24266 instruct vmul_reg_masked(vec dst, vec src2, kReg mask) %{
24267 match(Set dst (MulVS (Binary dst src2) mask));
24268 match(Set dst (MulVI (Binary dst src2) mask));
24269 match(Set dst (MulVL (Binary dst src2) mask));
24270 match(Set dst (MulVF (Binary dst src2) mask));
24271 match(Set dst (MulVD (Binary dst src2) mask));
24272 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
24273 ins_encode %{
24274 int vlen_enc = vector_length_encoding(this);
24275 BasicType bt = Matcher::vector_element_basic_type(this);
24276 int opc = this->ideal_Opcode();
24277 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24278 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24279 %}
24280 ins_pipe( pipe_slow );
24281 %}
24282
24283 instruct vmul_mem_masked(vec dst, memory src2, kReg mask) %{
24284 match(Set dst (MulVS (Binary dst (LoadVector src2)) mask));
24285 match(Set dst (MulVI (Binary dst (LoadVector src2)) mask));
24286 match(Set dst (MulVL (Binary dst (LoadVector src2)) mask));
24287 match(Set dst (MulVF (Binary dst (LoadVector src2)) mask));
24288 match(Set dst (MulVD (Binary dst (LoadVector src2)) mask));
24289 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
24290 ins_encode %{
24291 int vlen_enc = vector_length_encoding(this);
24292 BasicType bt = Matcher::vector_element_basic_type(this);
24293 int opc = this->ideal_Opcode();
24294 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24295 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24296 %}
24297 ins_pipe( pipe_slow );
24298 %}
24299
24300 instruct vsqrt_reg_masked(vec dst, kReg mask) %{
24301 match(Set dst (SqrtVF dst mask));
24302 match(Set dst (SqrtVD dst mask));
24303 format %{ "vpsqrt_masked $dst, $mask\t! sqrt masked operation" %}
24304 ins_encode %{
24305 int vlen_enc = vector_length_encoding(this);
24306 BasicType bt = Matcher::vector_element_basic_type(this);
24307 int opc = this->ideal_Opcode();
24308 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24309 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24310 %}
24311 ins_pipe( pipe_slow );
24312 %}
24313
24314 instruct vdiv_reg_masked(vec dst, vec src2, kReg mask) %{
24315 match(Set dst (DivVF (Binary dst src2) mask));
24316 match(Set dst (DivVD (Binary dst src2) mask));
24317 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24318 ins_encode %{
24319 int vlen_enc = vector_length_encoding(this);
24320 BasicType bt = Matcher::vector_element_basic_type(this);
24321 int opc = this->ideal_Opcode();
24322 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24323 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24324 %}
24325 ins_pipe( pipe_slow );
24326 %}
24327
24328 instruct vdiv_mem_masked(vec dst, memory src2, kReg mask) %{
24329 match(Set dst (DivVF (Binary dst (LoadVector src2)) mask));
24330 match(Set dst (DivVD (Binary dst (LoadVector src2)) mask));
24331 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24332 ins_encode %{
24333 int vlen_enc = vector_length_encoding(this);
24334 BasicType bt = Matcher::vector_element_basic_type(this);
24335 int opc = this->ideal_Opcode();
24336 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24337 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24338 %}
24339 ins_pipe( pipe_slow );
24340 %}
24341
24342
24343 instruct vrol_imm_masked(vec dst, immI8 shift, kReg mask) %{
24344 match(Set dst (RotateLeftV (Binary dst shift) mask));
24345 match(Set dst (RotateRightV (Binary dst shift) mask));
24346 format %{ "vprotate_imm_masked $dst, $dst, $shift, $mask\t! rotate masked operation" %}
24347 ins_encode %{
24348 int vlen_enc = vector_length_encoding(this);
24349 BasicType bt = Matcher::vector_element_basic_type(this);
24350 int opc = this->ideal_Opcode();
24351 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24352 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24353 %}
24354 ins_pipe( pipe_slow );
24355 %}
24356
24357 instruct vrol_reg_masked(vec dst, vec src2, kReg mask) %{
24358 match(Set dst (RotateLeftV (Binary dst src2) mask));
24359 match(Set dst (RotateRightV (Binary dst src2) mask));
24360 format %{ "vrotate_masked $dst, $dst, $src2, $mask\t! rotate masked operation" %}
24361 ins_encode %{
24362 int vlen_enc = vector_length_encoding(this);
24363 BasicType bt = Matcher::vector_element_basic_type(this);
24364 int opc = this->ideal_Opcode();
24365 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24366 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24367 %}
24368 ins_pipe( pipe_slow );
24369 %}
24370
24371 instruct vlshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24372 match(Set dst (LShiftVS (Binary dst (LShiftCntV shift)) mask));
24373 match(Set dst (LShiftVI (Binary dst (LShiftCntV shift)) mask));
24374 match(Set dst (LShiftVL (Binary dst (LShiftCntV shift)) mask));
24375 format %{ "vplshift_imm_masked $dst, $dst, $shift, $mask\t! lshift masked operation" %}
24376 ins_encode %{
24377 int vlen_enc = vector_length_encoding(this);
24378 BasicType bt = Matcher::vector_element_basic_type(this);
24379 int opc = this->ideal_Opcode();
24380 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24381 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24382 %}
24383 ins_pipe( pipe_slow );
24384 %}
24385
24386 instruct vlshift_reg_masked(vec dst, vec src2, kReg mask) %{
24387 predicate(!n->as_ShiftV()->is_var_shift());
24388 match(Set dst (LShiftVS (Binary dst src2) mask));
24389 match(Set dst (LShiftVI (Binary dst src2) mask));
24390 match(Set dst (LShiftVL (Binary dst src2) mask));
24391 format %{ "vplshift_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24392 ins_encode %{
24393 int vlen_enc = vector_length_encoding(this);
24394 BasicType bt = Matcher::vector_element_basic_type(this);
24395 int opc = this->ideal_Opcode();
24396 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24397 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24398 %}
24399 ins_pipe( pipe_slow );
24400 %}
24401
24402 instruct vlshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24403 predicate(n->as_ShiftV()->is_var_shift());
24404 match(Set dst (LShiftVS (Binary dst src2) mask));
24405 match(Set dst (LShiftVI (Binary dst src2) mask));
24406 match(Set dst (LShiftVL (Binary dst src2) mask));
24407 format %{ "vplshiftv_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24408 ins_encode %{
24409 int vlen_enc = vector_length_encoding(this);
24410 BasicType bt = Matcher::vector_element_basic_type(this);
24411 int opc = this->ideal_Opcode();
24412 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24413 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24414 %}
24415 ins_pipe( pipe_slow );
24416 %}
24417
24418 instruct vrshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24419 match(Set dst (RShiftVS (Binary dst (RShiftCntV shift)) mask));
24420 match(Set dst (RShiftVI (Binary dst (RShiftCntV shift)) mask));
24421 match(Set dst (RShiftVL (Binary dst (RShiftCntV shift)) mask));
24422 format %{ "vprshift_imm_masked $dst, $dst, $shift, $mask\t! rshift masked operation" %}
24423 ins_encode %{
24424 int vlen_enc = vector_length_encoding(this);
24425 BasicType bt = Matcher::vector_element_basic_type(this);
24426 int opc = this->ideal_Opcode();
24427 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24428 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24429 %}
24430 ins_pipe( pipe_slow );
24431 %}
24432
24433 instruct vrshift_reg_masked(vec dst, vec src2, kReg mask) %{
24434 predicate(!n->as_ShiftV()->is_var_shift());
24435 match(Set dst (RShiftVS (Binary dst src2) mask));
24436 match(Set dst (RShiftVI (Binary dst src2) mask));
24437 match(Set dst (RShiftVL (Binary dst src2) mask));
24438 format %{ "vprshift_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24439 ins_encode %{
24440 int vlen_enc = vector_length_encoding(this);
24441 BasicType bt = Matcher::vector_element_basic_type(this);
24442 int opc = this->ideal_Opcode();
24443 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24444 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24445 %}
24446 ins_pipe( pipe_slow );
24447 %}
24448
24449 instruct vrshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24450 predicate(n->as_ShiftV()->is_var_shift());
24451 match(Set dst (RShiftVS (Binary dst src2) mask));
24452 match(Set dst (RShiftVI (Binary dst src2) mask));
24453 match(Set dst (RShiftVL (Binary dst src2) mask));
24454 format %{ "vprshiftv_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24455 ins_encode %{
24456 int vlen_enc = vector_length_encoding(this);
24457 BasicType bt = Matcher::vector_element_basic_type(this);
24458 int opc = this->ideal_Opcode();
24459 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24460 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24461 %}
24462 ins_pipe( pipe_slow );
24463 %}
24464
24465 instruct vurshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24466 match(Set dst (URShiftVS (Binary dst (RShiftCntV shift)) mask));
24467 match(Set dst (URShiftVI (Binary dst (RShiftCntV shift)) mask));
24468 match(Set dst (URShiftVL (Binary dst (RShiftCntV shift)) mask));
24469 format %{ "vpurshift_imm_masked $dst, $dst, $shift, $mask\t! urshift masked operation" %}
24470 ins_encode %{
24471 int vlen_enc = vector_length_encoding(this);
24472 BasicType bt = Matcher::vector_element_basic_type(this);
24473 int opc = this->ideal_Opcode();
24474 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24475 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24476 %}
24477 ins_pipe( pipe_slow );
24478 %}
24479
24480 instruct vurshift_reg_masked(vec dst, vec src2, kReg mask) %{
24481 predicate(!n->as_ShiftV()->is_var_shift());
24482 match(Set dst (URShiftVS (Binary dst src2) mask));
24483 match(Set dst (URShiftVI (Binary dst src2) mask));
24484 match(Set dst (URShiftVL (Binary dst src2) mask));
24485 format %{ "vpurshift_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24486 ins_encode %{
24487 int vlen_enc = vector_length_encoding(this);
24488 BasicType bt = Matcher::vector_element_basic_type(this);
24489 int opc = this->ideal_Opcode();
24490 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24491 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24492 %}
24493 ins_pipe( pipe_slow );
24494 %}
24495
24496 instruct vurshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24497 predicate(n->as_ShiftV()->is_var_shift());
24498 match(Set dst (URShiftVS (Binary dst src2) mask));
24499 match(Set dst (URShiftVI (Binary dst src2) mask));
24500 match(Set dst (URShiftVL (Binary dst src2) mask));
24501 format %{ "vpurshiftv_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24502 ins_encode %{
24503 int vlen_enc = vector_length_encoding(this);
24504 BasicType bt = Matcher::vector_element_basic_type(this);
24505 int opc = this->ideal_Opcode();
24506 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24507 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24508 %}
24509 ins_pipe( pipe_slow );
24510 %}
24511
24512 instruct vmaxv_reg_masked(vec dst, vec src2, kReg mask) %{
24513 match(Set dst (MaxV (Binary dst src2) mask));
24514 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24515 ins_encode %{
24516 int vlen_enc = vector_length_encoding(this);
24517 BasicType bt = Matcher::vector_element_basic_type(this);
24518 int opc = this->ideal_Opcode();
24519 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24520 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24521 %}
24522 ins_pipe( pipe_slow );
24523 %}
24524
24525 instruct vmaxv_mem_masked(vec dst, memory src2, kReg mask) %{
24526 match(Set dst (MaxV (Binary dst (LoadVector src2)) mask));
24527 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24528 ins_encode %{
24529 int vlen_enc = vector_length_encoding(this);
24530 BasicType bt = Matcher::vector_element_basic_type(this);
24531 int opc = this->ideal_Opcode();
24532 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24533 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24534 %}
24535 ins_pipe( pipe_slow );
24536 %}
24537
24538 instruct vminv_reg_masked(vec dst, vec src2, kReg mask) %{
24539 match(Set dst (MinV (Binary dst src2) mask));
24540 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24541 ins_encode %{
24542 int vlen_enc = vector_length_encoding(this);
24543 BasicType bt = Matcher::vector_element_basic_type(this);
24544 int opc = this->ideal_Opcode();
24545 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24546 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24547 %}
24548 ins_pipe( pipe_slow );
24549 %}
24550
24551 instruct vminv_mem_masked(vec dst, memory src2, kReg mask) %{
24552 match(Set dst (MinV (Binary dst (LoadVector src2)) mask));
24553 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24554 ins_encode %{
24555 int vlen_enc = vector_length_encoding(this);
24556 BasicType bt = Matcher::vector_element_basic_type(this);
24557 int opc = this->ideal_Opcode();
24558 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24559 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24560 %}
24561 ins_pipe( pipe_slow );
24562 %}
24563
24564 instruct vrearrangev_reg_masked(vec dst, vec src2, kReg mask) %{
24565 match(Set dst (VectorRearrange (Binary dst src2) mask));
24566 format %{ "vprearrange_masked $dst, $dst, $src2, $mask\t! rearrange masked operation" %}
24567 ins_encode %{
24568 int vlen_enc = vector_length_encoding(this);
24569 BasicType bt = Matcher::vector_element_basic_type(this);
24570 int opc = this->ideal_Opcode();
24571 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24572 $dst$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24573 %}
24574 ins_pipe( pipe_slow );
24575 %}
24576
24577 instruct vabs_masked(vec dst, kReg mask) %{
24578 match(Set dst (AbsVB dst mask));
24579 match(Set dst (AbsVS dst mask));
24580 match(Set dst (AbsVI dst mask));
24581 match(Set dst (AbsVL dst mask));
24582 format %{ "vabs_masked $dst, $mask \t! vabs masked operation" %}
24583 ins_encode %{
24584 int vlen_enc = vector_length_encoding(this);
24585 BasicType bt = Matcher::vector_element_basic_type(this);
24586 int opc = this->ideal_Opcode();
24587 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24588 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24589 %}
24590 ins_pipe( pipe_slow );
24591 %}
24592
24593 instruct vfma_reg_masked(vec dst, vec src2, vec src3, kReg mask) %{
24594 match(Set dst (FmaVF (Binary dst src2) (Binary src3 mask)));
24595 match(Set dst (FmaVD (Binary dst src2) (Binary src3 mask)));
24596 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24597 ins_encode %{
24598 assert(UseFMA, "Needs FMA instructions support.");
24599 int vlen_enc = vector_length_encoding(this);
24600 BasicType bt = Matcher::vector_element_basic_type(this);
24601 int opc = this->ideal_Opcode();
24602 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24603 $src2$$XMMRegister, $src3$$XMMRegister, true, vlen_enc);
24604 %}
24605 ins_pipe( pipe_slow );
24606 %}
24607
24608 instruct vfma_mem_masked(vec dst, vec src2, memory src3, kReg mask) %{
24609 match(Set dst (FmaVF (Binary dst src2) (Binary (LoadVector src3) mask)));
24610 match(Set dst (FmaVD (Binary dst src2) (Binary (LoadVector src3) mask)));
24611 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24612 ins_encode %{
24613 assert(UseFMA, "Needs FMA instructions support.");
24614 int vlen_enc = vector_length_encoding(this);
24615 BasicType bt = Matcher::vector_element_basic_type(this);
24616 int opc = this->ideal_Opcode();
24617 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24618 $src2$$XMMRegister, $src3$$Address, true, vlen_enc);
24619 %}
24620 ins_pipe( pipe_slow );
24621 %}
24622
24623 instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
24624 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
24625 format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
24626 ins_encode %{
24627 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
24628 int vlen_enc = vector_length_encoding(this, $src1);
24629 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
24630
24631 // Comparison i
24632 switch (src1_elem_bt) {
24633 case T_BYTE: {
24634 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24635 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24636 __ evpcmpb($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24637 break;
24638 }
24639 case T_SHORT: {
24640 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24641 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24642 __ evpcmpw($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24643 break;
24644 }
24645 case T_INT: {
24646 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24647 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24648 __ evpcmpd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24649 break;
24650 }
24651 case T_LONG: {
24652 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24653 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24654 __ evpcmpq($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24655 break;
24656 }
24657 case T_FLOAT: {
24658 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24659 __ evcmpps($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24660 break;
24661 }
24662 case T_DOUBLE: {
24663 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24664 __ evcmppd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24665 break;
24666 }
24667 default: assert(false, "%s", type2name(src1_elem_bt)); break;
24668 }
24669 %}
24670 ins_pipe( pipe_slow );
24671 %}
24672
24673 instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
24674 predicate(Matcher::vector_length(n) <= 32);
24675 match(Set dst (MaskAll src));
24676 format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
24677 ins_encode %{
24678 int mask_len = Matcher::vector_length(this);
24679 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
24680 %}
24681 ins_pipe( pipe_slow );
24682 %}
24683
24684 instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
24685 predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
24686 match(Set dst (XorVMask src (MaskAll cnt)));
24687 effect(TEMP_DEF dst, TEMP rtmp, TEMP ktmp);
24688 format %{ "mask_not_LT8 $dst, $src, $cnt \t!using $ktmp and $rtmp as TEMP" %}
24689 ins_encode %{
24690 uint masklen = Matcher::vector_length(this);
24691 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
24692 %}
24693 ins_pipe( pipe_slow );
24694 %}
24695
24696 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
24697 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
24698 (Matcher::vector_length(n) == 16) ||
24699 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
24700 match(Set dst (XorVMask src (MaskAll cnt)));
24701 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
24702 ins_encode %{
24703 uint masklen = Matcher::vector_length(this);
24704 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
24705 %}
24706 ins_pipe( pipe_slow );
24707 %}
24708
24709 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
24710 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
24711 match(Set dst (VectorLongToMask src));
24712 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
24713 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
24714 ins_encode %{
24715 int mask_len = Matcher::vector_length(this);
24716 int vec_enc = vector_length_encoding(mask_len);
24717 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24718 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
24719 %}
24720 ins_pipe( pipe_slow );
24721 %}
24722
24723
24724 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
24725 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
24726 match(Set dst (VectorLongToMask src));
24727 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
24728 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
24729 ins_encode %{
24730 int mask_len = Matcher::vector_length(this);
24731 assert(mask_len <= 32, "invalid mask length");
24732 int vec_enc = vector_length_encoding(mask_len);
24733 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24734 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
24735 %}
24736 ins_pipe( pipe_slow );
24737 %}
24738
24739 instruct long_to_mask_evex(kReg dst, rRegL src) %{
24740 predicate(n->bottom_type()->isa_pvectmask());
24741 match(Set dst (VectorLongToMask src));
24742 format %{ "long_to_mask_evex $dst, $src\t!" %}
24743 ins_encode %{
24744 __ kmov($dst$$KRegister, $src$$Register);
24745 %}
24746 ins_pipe( pipe_slow );
24747 %}
24748
24749 instruct mask_opers_evex(kReg dst, kReg src1, kReg src2, kReg kscratch) %{
24750 match(Set dst (AndVMask src1 src2));
24751 match(Set dst (OrVMask src1 src2));
24752 match(Set dst (XorVMask src1 src2));
24753 effect(TEMP kscratch);
24754 format %{ "mask_opers_evex $dst, $src1, $src2\t! using $kscratch as TEMP" %}
24755 ins_encode %{
24756 const MachNode* mask1 = static_cast<const MachNode*>(this->in(this->operand_index($src1)));
24757 const MachNode* mask2 = static_cast<const MachNode*>(this->in(this->operand_index($src2)));
24758 assert(Type::equals(mask1->bottom_type(), mask2->bottom_type()), "Mask types must be equal");
24759 uint masklen = Matcher::vector_length(this);
24760 masklen = (masklen < 16 && !VM_Version::supports_avx512dq()) ? 16 : masklen;
24761 __ masked_op(this->ideal_Opcode(), masklen, $dst$$KRegister, $src1$$KRegister, $src2$$KRegister);
24762 %}
24763 ins_pipe( pipe_slow );
24764 %}
24765
24766 instruct vternlog_reg_masked(vec dst, vec src2, vec src3, immU8 func, kReg mask) %{
24767 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24768 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24769 ins_encode %{
24770 int vlen_enc = vector_length_encoding(this);
24771 BasicType bt = Matcher::vector_element_basic_type(this);
24772 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24773 $src2$$XMMRegister, $src3$$XMMRegister, true, bt, vlen_enc);
24774 %}
24775 ins_pipe( pipe_slow );
24776 %}
24777
24778 instruct vternlogd_mem_masked(vec dst, vec src2, memory src3, immU8 func, kReg mask) %{
24779 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24780 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24781 ins_encode %{
24782 int vlen_enc = vector_length_encoding(this);
24783 BasicType bt = Matcher::vector_element_basic_type(this);
24784 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24785 $src2$$XMMRegister, $src3$$Address, true, bt, vlen_enc);
24786 %}
24787 ins_pipe( pipe_slow );
24788 %}
24789
24790 instruct castMM(kReg dst)
24791 %{
24792 match(Set dst (CastVV dst));
24793
24794 size(0);
24795 format %{ "# castVV of $dst" %}
24796 ins_encode(/* empty encoding */);
24797 ins_cost(0);
24798 ins_pipe(empty);
24799 %}
24800
24801 instruct castVV(vec dst)
24802 %{
24803 match(Set dst (CastVV dst));
24804
24805 size(0);
24806 format %{ "# castVV of $dst" %}
24807 ins_encode(/* empty encoding */);
24808 ins_cost(0);
24809 ins_pipe(empty);
24810 %}
24811
24812 instruct castVVLeg(legVec dst)
24813 %{
24814 match(Set dst (CastVV dst));
24815
24816 size(0);
24817 format %{ "# castVV of $dst" %}
24818 ins_encode(/* empty encoding */);
24819 ins_cost(0);
24820 ins_pipe(empty);
24821 %}
24822
24823 instruct FloatClassCheck_reg_reg_vfpclass(rRegI dst, regF src, kReg ktmp, rFlagsReg cr)
24824 %{
24825 match(Set dst (IsInfiniteF src));
24826 effect(TEMP ktmp, KILL cr);
24827 format %{ "float_class_check $dst, $src" %}
24828 ins_encode %{
24829 __ vfpclassss($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24830 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24831 %}
24832 ins_pipe(pipe_slow);
24833 %}
24834
24835 instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlagsReg cr)
24836 %{
24837 match(Set dst (IsInfiniteD src));
24838 effect(TEMP ktmp, KILL cr);
24839 format %{ "double_class_check $dst, $src" %}
24840 ins_encode %{
24841 __ vfpclasssd($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24842 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24843 %}
24844 ins_pipe(pipe_slow);
24845 %}
24846
24847 instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
24848 %{
24849 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24850 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24851 match(Set dst (SaturatingAddV src1 src2));
24852 match(Set dst (SaturatingSubV src1 src2));
24853 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24854 ins_encode %{
24855 int vlen_enc = vector_length_encoding(this);
24856 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24857 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24858 $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24859 %}
24860 ins_pipe(pipe_slow);
24861 %}
24862
24863 instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
24864 %{
24865 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24866 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24867 match(Set dst (SaturatingAddV src1 src2));
24868 match(Set dst (SaturatingSubV src1 src2));
24869 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24870 ins_encode %{
24871 int vlen_enc = vector_length_encoding(this);
24872 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24873 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24874 $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24875 %}
24876 ins_pipe(pipe_slow);
24877 %}
24878
24879 instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
24880 %{
24881 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24882 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24883 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24884 match(Set dst (SaturatingAddV src1 src2));
24885 match(Set dst (SaturatingSubV src1 src2));
24886 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
24887 format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
24888 ins_encode %{
24889 int vlen_enc = vector_length_encoding(this);
24890 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24891 __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24892 $src1$$XMMRegister, $src2$$XMMRegister,
24893 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24894 $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
24895 %}
24896 ins_pipe(pipe_slow);
24897 %}
24898
24899 instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
24900 %{
24901 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24902 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24903 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24904 match(Set dst (SaturatingAddV src1 src2));
24905 match(Set dst (SaturatingSubV src1 src2));
24906 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
24907 format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
24908 ins_encode %{
24909 int vlen_enc = vector_length_encoding(this);
24910 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24911 __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24912 $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24913 $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
24914 %}
24915 ins_pipe(pipe_slow);
24916 %}
24917
24918 instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
24919 %{
24920 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24921 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24922 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24923 match(Set dst (SaturatingAddV src1 src2));
24924 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
24925 format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
24926 ins_encode %{
24927 int vlen_enc = vector_length_encoding(this);
24928 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24929 __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24930 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24931 %}
24932 ins_pipe(pipe_slow);
24933 %}
24934
24935 instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
24936 %{
24937 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24938 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24939 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24940 match(Set dst (SaturatingAddV src1 src2));
24941 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24942 format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24943 ins_encode %{
24944 int vlen_enc = vector_length_encoding(this);
24945 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24946 __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24947 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
24948 %}
24949 ins_pipe(pipe_slow);
24950 %}
24951
24952 instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
24953 %{
24954 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24955 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24956 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24957 match(Set dst (SaturatingSubV src1 src2));
24958 effect(TEMP ktmp);
24959 format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
24960 ins_encode %{
24961 int vlen_enc = vector_length_encoding(this);
24962 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24963 __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24964 $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24965 %}
24966 ins_pipe(pipe_slow);
24967 %}
24968
24969 instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
24970 %{
24971 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24972 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24973 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24974 match(Set dst (SaturatingSubV src1 src2));
24975 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24976 format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
24977 ins_encode %{
24978 int vlen_enc = vector_length_encoding(this);
24979 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24980 __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24981 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24982 %}
24983 ins_pipe(pipe_slow);
24984 %}
24985
24986 instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
24987 %{
24988 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24989 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24990 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24991 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24992 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24993 ins_encode %{
24994 int vlen_enc = vector_length_encoding(this);
24995 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24996 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24997 $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
24998 %}
24999 ins_pipe(pipe_slow);
25000 %}
25001
25002 instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
25003 %{
25004 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25005 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25006 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
25007 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
25008 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
25009 ins_encode %{
25010 int vlen_enc = vector_length_encoding(this);
25011 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25012 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
25013 $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
25014 %}
25015 ins_pipe(pipe_slow);
25016 %}
25017
25018 instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
25019 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25020 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
25021 match(Set dst (SaturatingAddV (Binary dst src) mask));
25022 match(Set dst (SaturatingSubV (Binary dst src) mask));
25023 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
25024 ins_encode %{
25025 int vlen_enc = vector_length_encoding(this);
25026 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25027 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25028 $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
25029 %}
25030 ins_pipe( pipe_slow );
25031 %}
25032
25033 instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
25034 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25035 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25036 match(Set dst (SaturatingAddV (Binary dst src) mask));
25037 match(Set dst (SaturatingSubV (Binary dst src) mask));
25038 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
25039 ins_encode %{
25040 int vlen_enc = vector_length_encoding(this);
25041 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25042 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25043 $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
25044 %}
25045 ins_pipe( pipe_slow );
25046 %}
25047
25048 instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
25049 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25050 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
25051 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
25052 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
25053 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
25054 ins_encode %{
25055 int vlen_enc = vector_length_encoding(this);
25056 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25057 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25058 $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
25059 %}
25060 ins_pipe( pipe_slow );
25061 %}
25062
25063 instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
25064 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25065 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25066 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
25067 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
25068 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
25069 ins_encode %{
25070 int vlen_enc = vector_length_encoding(this);
25071 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25072 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25073 $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
25074 %}
25075 ins_pipe( pipe_slow );
25076 %}
25077
25078 instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
25079 %{
25080 match(Set index (SelectFromTwoVector (Binary index src1) src2));
25081 format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
25082 ins_encode %{
25083 int vlen_enc = vector_length_encoding(this);
25084 BasicType bt = Matcher::vector_element_basic_type(this);
25085 __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
25086 %}
25087 ins_pipe(pipe_slow);
25088 %}
25089
25090 instruct reinterpretS2HF(regF dst, rRegI src)
25091 %{
25092 match(Set dst (ReinterpretS2HF src));
25093 format %{ "evmovw $dst, $src" %}
25094 ins_encode %{
25095 __ evmovw($dst$$XMMRegister, $src$$Register);
25096 %}
25097 ins_pipe(pipe_slow);
25098 %}
25099
25100 instruct reinterpretHF2S(rRegI dst, regF src)
25101 %{
25102 match(Set dst (ReinterpretHF2S src));
25103 format %{ "evmovw $dst, $src" %}
25104 ins_encode %{
25105 __ evmovw($dst$$Register, $src$$XMMRegister);
25106 __ narrow_subword_type($dst$$Register, T_SHORT);
25107 %}
25108 ins_pipe(pipe_slow);
25109 %}
25110
25111 instruct convF2HFAndS2HF(regF dst, regF src)
25112 %{
25113 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
25114 format %{ "convF2HFAndS2HF $dst, $src" %}
25115 ins_encode %{
25116 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
25117 %}
25118 ins_pipe(pipe_slow);
25119 %}
25120
25121 instruct convHF2SAndHF2F(regF dst, regF src)
25122 %{
25123 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
25124 format %{ "convHF2SAndHF2F $dst, $src" %}
25125 ins_encode %{
25126 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, Assembler::AVX_128bit);
25127 %}
25128 ins_pipe(pipe_slow);
25129 %}
25130
25131 instruct scalar_sqrt_HF_reg(regF dst, regF src)
25132 %{
25133 match(Set dst (SqrtHF src));
25134 format %{ "scalar_sqrt_fp16 $dst, $src" %}
25135 ins_encode %{
25136 __ vsqrtsh($dst$$XMMRegister, $src$$XMMRegister);
25137 %}
25138 ins_pipe(pipe_slow);
25139 %}
25140
25141 instruct scalar_binOps_HF_reg(regF dst, regF src1, regF src2)
25142 %{
25143 match(Set dst (AddHF src1 src2));
25144 match(Set dst (DivHF src1 src2));
25145 match(Set dst (MulHF src1 src2));
25146 match(Set dst (SubHF src1 src2));
25147 format %{ "scalar_binop_fp16 $dst, $src1, $src2" %}
25148 ins_encode %{
25149 int opcode = this->ideal_Opcode();
25150 __ efp16sh(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
25151 %}
25152 ins_pipe(pipe_slow);
25153 %}
25154
25155 instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
25156 %{
25157 predicate(VM_Version::supports_avx10_2());
25158 match(Set dst (MaxHF src1 src2));
25159 match(Set dst (MinHF src1 src2));
25160
25161 format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
25162 ins_encode %{
25163 int opcode = this->ideal_Opcode();
25164 __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
25165 %}
25166 ins_pipe( pipe_slow );
25167 %}
25168
25169 instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xtmp1, regF xtmp2)
25170 %{
25171 predicate(!VM_Version::supports_avx10_2());
25172 match(Set dst (MaxHF src1 src2));
25173 match(Set dst (MinHF src1 src2));
25174 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
25175
25176 format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
25177 ins_encode %{
25178 int opcode = this->ideal_Opcode();
25179 __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
25180 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
25181 %}
25182 ins_pipe( pipe_slow );
25183 %}
25184
25185 instruct scalar_fma_HF_reg(regF dst, regF src1, regF src2)
25186 %{
25187 match(Set dst (FmaHF src2 (Binary dst src1)));
25188 effect(DEF dst);
25189 format %{ "scalar_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25190 ins_encode %{
25191 __ vfmadd132sh($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister);
25192 %}
25193 ins_pipe( pipe_slow );
25194 %}
25195
25196
25197 instruct vector_sqrt_HF_reg(vec dst, vec src)
25198 %{
25199 match(Set dst (SqrtVHF src));
25200 format %{ "vector_sqrt_fp16 $dst, $src" %}
25201 ins_encode %{
25202 int vlen_enc = vector_length_encoding(this);
25203 __ evsqrtph($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
25204 %}
25205 ins_pipe(pipe_slow);
25206 %}
25207
25208 instruct vector_sqrt_HF_mem(vec dst, memory src)
25209 %{
25210 match(Set dst (SqrtVHF (VectorReinterpret (LoadVector src))));
25211 format %{ "vector_sqrt_fp16_mem $dst, $src" %}
25212 ins_encode %{
25213 int vlen_enc = vector_length_encoding(this);
25214 __ evsqrtph($dst$$XMMRegister, $src$$Address, vlen_enc);
25215 %}
25216 ins_pipe(pipe_slow);
25217 %}
25218
25219 instruct vector_binOps_HF_reg(vec dst, vec src1, vec src2)
25220 %{
25221 match(Set dst (AddVHF src1 src2));
25222 match(Set dst (DivVHF src1 src2));
25223 match(Set dst (MulVHF src1 src2));
25224 match(Set dst (SubVHF src1 src2));
25225 format %{ "vector_binop_fp16 $dst, $src1, $src2" %}
25226 ins_encode %{
25227 int vlen_enc = vector_length_encoding(this);
25228 int opcode = this->ideal_Opcode();
25229 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
25230 %}
25231 ins_pipe(pipe_slow);
25232 %}
25233
25234
25235 instruct vector_binOps_HF_mem(vec dst, vec src1, memory src2)
25236 %{
25237 match(Set dst (AddVHF src1 (VectorReinterpret (LoadVector src2))));
25238 match(Set dst (DivVHF src1 (VectorReinterpret (LoadVector src2))));
25239 match(Set dst (MulVHF src1 (VectorReinterpret (LoadVector src2))));
25240 match(Set dst (SubVHF src1 (VectorReinterpret (LoadVector src2))));
25241 format %{ "vector_binop_fp16_mem $dst, $src1, $src2" %}
25242 ins_encode %{
25243 int vlen_enc = vector_length_encoding(this);
25244 int opcode = this->ideal_Opcode();
25245 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address, vlen_enc);
25246 %}
25247 ins_pipe(pipe_slow);
25248 %}
25249
25250 instruct vector_fma_HF_reg(vec dst, vec src1, vec src2)
25251 %{
25252 match(Set dst (FmaVHF src2 (Binary dst src1)));
25253 format %{ "vector_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25254 ins_encode %{
25255 int vlen_enc = vector_length_encoding(this);
25256 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vlen_enc);
25257 %}
25258 ins_pipe( pipe_slow );
25259 %}
25260
25261 instruct vector_fma_HF_mem(vec dst, memory src1, vec src2)
25262 %{
25263 match(Set dst (FmaVHF src2 (Binary dst (VectorReinterpret (LoadVector src1)))));
25264 format %{ "vector_fma_fp16_mem $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25265 ins_encode %{
25266 int vlen_enc = vector_length_encoding(this);
25267 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$Address, vlen_enc);
25268 %}
25269 ins_pipe( pipe_slow );
25270 %}
25271
25272 instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
25273 %{
25274 predicate(VM_Version::supports_avx10_2());
25275 match(Set dst (MinVHF src1 (VectorReinterpret (LoadVector src2))));
25276 match(Set dst (MaxVHF src1 (VectorReinterpret (LoadVector src2))));
25277 format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
25278 ins_encode %{
25279 int vlen_enc = vector_length_encoding(this);
25280 int opcode = this->ideal_Opcode();
25281 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
25282 k0, vlen_enc);
25283 %}
25284 ins_pipe( pipe_slow );
25285 %}
25286
25287 instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
25288 %{
25289 predicate(VM_Version::supports_avx10_2());
25290 match(Set dst (MinVHF src1 src2));
25291 match(Set dst (MaxVHF src1 src2));
25292 format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
25293 ins_encode %{
25294 int vlen_enc = vector_length_encoding(this);
25295 int opcode = this->ideal_Opcode();
25296 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
25297 k0, vlen_enc);
25298 %}
25299 ins_pipe( pipe_slow );
25300 %}
25301
25302 instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1, vec xtmp2)
25303 %{
25304 predicate(!VM_Version::supports_avx10_2());
25305 match(Set dst (MinVHF src1 src2));
25306 match(Set dst (MaxVHF src1 src2));
25307 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
25308 format %{ "vector_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
25309 ins_encode %{
25310 int vlen_enc = vector_length_encoding(this);
25311 int opcode = this->ideal_Opcode();
25312 __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
25313 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
25314 %}
25315 ins_pipe( pipe_slow );
25316 %}
25317
25318 //----------PEEPHOLE RULES-----------------------------------------------------
25319 // These must follow all instruction definitions as they use the names
25320 // defined in the instructions definitions.
25321 //
25322 // peeppredicate ( rule_predicate );
25323 // // the predicate unless which the peephole rule will be ignored
25324 //
25325 // peepmatch ( root_instr_name [preceding_instruction]* );
25326 //
25327 // peepprocedure ( procedure_name );
25328 // // provide a procedure name to perform the optimization, the procedure should
25329 // // reside in the architecture dependent peephole file, the method has the
25330 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
25331 // // with the arguments being the basic block, the current node index inside the
25332 // // block, the register allocator, the functions upon invoked return a new node
25333 // // defined in peepreplace, and the rules of the nodes appearing in the
25334 // // corresponding peepmatch, the function return true if successful, else
25335 // // return false
25336 //
25337 // peepconstraint %{
25338 // (instruction_number.operand_name relational_op instruction_number.operand_name
25339 // [, ...] );
25340 // // instruction numbers are zero-based using left to right order in peepmatch
25341 //
25342 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
25343 // // provide an instruction_number.operand_name for each operand that appears
25344 // // in the replacement instruction's match rule
25345 //
25346 // ---------VM FLAGS---------------------------------------------------------
25347 //
25348 // All peephole optimizations can be turned off using -XX:-OptoPeephole
25349 //
25350 // Each peephole rule is given an identifying number starting with zero and
25351 // increasing by one in the order seen by the parser. An individual peephole
25352 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
25353 // on the command-line.
25354 //
25355 // ---------CURRENT LIMITATIONS----------------------------------------------
25356 //
25357 // Only transformations inside a basic block (do we need more for peephole)
25358 //
25359 // ---------EXAMPLE----------------------------------------------------------
25360 //
25361 // // pertinent parts of existing instructions in architecture description
25362 // instruct movI(rRegI dst, rRegI src)
25363 // %{
25364 // match(Set dst (CopyI src));
25365 // %}
25366 //
25367 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
25368 // %{
25369 // match(Set dst (AddI dst src));
25370 // effect(KILL cr);
25371 // %}
25372 //
25373 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
25374 // %{
25375 // match(Set dst (AddI dst src));
25376 // %}
25377 //
25378 // 1. Simple replacement
25379 // - Only match adjacent instructions in same basic block
25380 // - Only equality constraints
25381 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
25382 // - Only one replacement instruction
25383 //
25384 // // Change (inc mov) to lea
25385 // peephole %{
25386 // // lea should only be emitted when beneficial
25387 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25388 // // increment preceded by register-register move
25389 // peepmatch ( incI_rReg movI );
25390 // // require that the destination register of the increment
25391 // // match the destination register of the move
25392 // peepconstraint ( 0.dst == 1.dst );
25393 // // construct a replacement instruction that sets
25394 // // the destination to ( move's source register + one )
25395 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
25396 // %}
25397 //
25398 // 2. Procedural replacement
25399 // - More flexible finding relevent nodes
25400 // - More flexible constraints
25401 // - More flexible transformations
25402 // - May utilise architecture-dependent API more effectively
25403 // - Currently only one replacement instruction due to adlc parsing capabilities
25404 //
25405 // // Change (inc mov) to lea
25406 // peephole %{
25407 // // lea should only be emitted when beneficial
25408 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25409 // // the rule numbers of these nodes inside are passed into the function below
25410 // peepmatch ( incI_rReg movI );
25411 // // the method that takes the responsibility of transformation
25412 // peepprocedure ( inc_mov_to_lea );
25413 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
25414 // // node is passed into the function above
25415 // peepreplace ( leaI_rReg_immI() );
25416 // %}
25417
25418 // These instructions is not matched by the matcher but used by the peephole
25419 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
25420 %{
25421 predicate(false);
25422 match(Set dst (AddI src1 src2));
25423 format %{ "leal $dst, [$src1 + $src2]" %}
25424 ins_encode %{
25425 Register dst = $dst$$Register;
25426 Register src1 = $src1$$Register;
25427 Register src2 = $src2$$Register;
25428 if (src1 != rbp && src1 != r13) {
25429 __ leal(dst, Address(src1, src2, Address::times_1));
25430 } else {
25431 assert(src2 != rbp && src2 != r13, "");
25432 __ leal(dst, Address(src2, src1, Address::times_1));
25433 }
25434 %}
25435 ins_pipe(ialu_reg_reg);
25436 %}
25437
25438 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
25439 %{
25440 predicate(false);
25441 match(Set dst (AddI src1 src2));
25442 format %{ "leal $dst, [$src1 + $src2]" %}
25443 ins_encode %{
25444 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
25445 %}
25446 ins_pipe(ialu_reg_reg);
25447 %}
25448
25449 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
25450 %{
25451 predicate(false);
25452 match(Set dst (LShiftI src shift));
25453 format %{ "leal $dst, [$src << $shift]" %}
25454 ins_encode %{
25455 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25456 Register src = $src$$Register;
25457 if (scale == Address::times_2 && src != rbp && src != r13) {
25458 __ leal($dst$$Register, Address(src, src, Address::times_1));
25459 } else {
25460 __ leal($dst$$Register, Address(noreg, src, scale));
25461 }
25462 %}
25463 ins_pipe(ialu_reg_reg);
25464 %}
25465
25466 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
25467 %{
25468 predicate(false);
25469 match(Set dst (AddL src1 src2));
25470 format %{ "leaq $dst, [$src1 + $src2]" %}
25471 ins_encode %{
25472 Register dst = $dst$$Register;
25473 Register src1 = $src1$$Register;
25474 Register src2 = $src2$$Register;
25475 if (src1 != rbp && src1 != r13) {
25476 __ leaq(dst, Address(src1, src2, Address::times_1));
25477 } else {
25478 assert(src2 != rbp && src2 != r13, "");
25479 __ leaq(dst, Address(src2, src1, Address::times_1));
25480 }
25481 %}
25482 ins_pipe(ialu_reg_reg);
25483 %}
25484
25485 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
25486 %{
25487 predicate(false);
25488 match(Set dst (AddL src1 src2));
25489 format %{ "leaq $dst, [$src1 + $src2]" %}
25490 ins_encode %{
25491 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
25492 %}
25493 ins_pipe(ialu_reg_reg);
25494 %}
25495
25496 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
25497 %{
25498 predicate(false);
25499 match(Set dst (LShiftL src shift));
25500 format %{ "leaq $dst, [$src << $shift]" %}
25501 ins_encode %{
25502 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25503 Register src = $src$$Register;
25504 if (scale == Address::times_2 && src != rbp && src != r13) {
25505 __ leaq($dst$$Register, Address(src, src, Address::times_1));
25506 } else {
25507 __ leaq($dst$$Register, Address(noreg, src, scale));
25508 }
25509 %}
25510 ins_pipe(ialu_reg_reg);
25511 %}
25512
25513 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
25514 // sal}) with lea instructions. The {add, sal} rules are beneficial in
25515 // processors with at least partial ALU support for lea
25516 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
25517 // beneficial for processors with full ALU support
25518 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
25519
25520 peephole
25521 %{
25522 peeppredicate(VM_Version::supports_fast_2op_lea());
25523 peepmatch (addI_rReg);
25524 peepprocedure (lea_coalesce_reg);
25525 peepreplace (leaI_rReg_rReg_peep());
25526 %}
25527
25528 peephole
25529 %{
25530 peeppredicate(VM_Version::supports_fast_2op_lea());
25531 peepmatch (addI_rReg_imm);
25532 peepprocedure (lea_coalesce_imm);
25533 peepreplace (leaI_rReg_immI_peep());
25534 %}
25535
25536 peephole
25537 %{
25538 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25539 VM_Version::is_intel_cascade_lake());
25540 peepmatch (incI_rReg);
25541 peepprocedure (lea_coalesce_imm);
25542 peepreplace (leaI_rReg_immI_peep());
25543 %}
25544
25545 peephole
25546 %{
25547 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25548 VM_Version::is_intel_cascade_lake());
25549 peepmatch (decI_rReg);
25550 peepprocedure (lea_coalesce_imm);
25551 peepreplace (leaI_rReg_immI_peep());
25552 %}
25553
25554 peephole
25555 %{
25556 peeppredicate(VM_Version::supports_fast_2op_lea());
25557 peepmatch (salI_rReg_immI2);
25558 peepprocedure (lea_coalesce_imm);
25559 peepreplace (leaI_rReg_immI2_peep());
25560 %}
25561
25562 peephole
25563 %{
25564 peeppredicate(VM_Version::supports_fast_2op_lea());
25565 peepmatch (addL_rReg);
25566 peepprocedure (lea_coalesce_reg);
25567 peepreplace (leaL_rReg_rReg_peep());
25568 %}
25569
25570 peephole
25571 %{
25572 peeppredicate(VM_Version::supports_fast_2op_lea());
25573 peepmatch (addL_rReg_imm);
25574 peepprocedure (lea_coalesce_imm);
25575 peepreplace (leaL_rReg_immL32_peep());
25576 %}
25577
25578 peephole
25579 %{
25580 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25581 VM_Version::is_intel_cascade_lake());
25582 peepmatch (incL_rReg);
25583 peepprocedure (lea_coalesce_imm);
25584 peepreplace (leaL_rReg_immL32_peep());
25585 %}
25586
25587 peephole
25588 %{
25589 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25590 VM_Version::is_intel_cascade_lake());
25591 peepmatch (decL_rReg);
25592 peepprocedure (lea_coalesce_imm);
25593 peepreplace (leaL_rReg_immL32_peep());
25594 %}
25595
25596 peephole
25597 %{
25598 peeppredicate(VM_Version::supports_fast_2op_lea());
25599 peepmatch (salL_rReg_immI2);
25600 peepprocedure (lea_coalesce_imm);
25601 peepreplace (leaL_rReg_immI2_peep());
25602 %}
25603
25604 peephole
25605 %{
25606 peepmatch (leaPCompressedOopOffset);
25607 peepprocedure (lea_remove_redundant);
25608 %}
25609
25610 peephole
25611 %{
25612 peepmatch (leaP8Narrow);
25613 peepprocedure (lea_remove_redundant);
25614 %}
25615
25616 peephole
25617 %{
25618 peepmatch (leaP32Narrow);
25619 peepprocedure (lea_remove_redundant);
25620 %}
25621
25622 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
25623 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
25624
25625 //int variant
25626 peephole
25627 %{
25628 peepmatch (testI_reg);
25629 peepprocedure (test_may_remove);
25630 %}
25631
25632 //long variant
25633 peephole
25634 %{
25635 peepmatch (testL_reg);
25636 peepprocedure (test_may_remove);
25637 %}
25638
25639
25640 //----------SMARTSPILL RULES---------------------------------------------------
25641 // These must follow all instruction definitions as they use the names
25642 // defined in the instructions definitions.